leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 29 Packages Projects Releases Wiki Activity

CUTLASS 2.0 (#62)

Browse Source 
CUTLASS 2.0

Substantially refactored for

- Better performance, particularly for native Turing Tensor Cores
- Robust and durable templates spanning the design space
- Encapsulated functionality embodying modern C++11 programming techniques
- Optimized containers and data types for efficient, generic, portable device code

Updates to:
- Quick start guide
- Documentation
- Utilities
- CUTLASS Profiler

Native Turing Tensor Cores
- Efficient GEMM kernels targeting Turing Tensor Cores
- Mixed-precision floating point, 8-bit integer, 4-bit integer, and binarized operands

Coverage of existing CUTLASS functionality:
- GEMM kernels targeting CUDA and Tensor Cores in NVIDIA GPUs
- Volta Tensor Cores through native mma.sync and through WMMA API
- Optimizations such as parallel reductions, threadblock rasterization, and intra-threadblock reductions
- Batched GEMM operations
- Complex-valued GEMMs

Note: this commit and all that follow require a host compiler supporting C++11 or greater.
This commit is contained in:
Andrew Kerr
2019-11-19 16:55:34 -08:00
committed by GitHub
parent b5cab177a9
commit fb335f6a5f
5434 changed files with 599799 additions and 250176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/00_basic_gemm/CMakeLists.txt
+3 -14
View File
@@ -20,19 +20,8 @@
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
set(EXAMPLES_BASIC_CUTLASS_GEMM_SOURCES
cutlass_example_add_executable(
00_basic_gemm
basic_gemm.cu
)
if (NOT CUTLASS_NATIVE_CUDA)
# cuda_add_executable does not take interface include directories into account
# Let's fetch them and pass them to CUDA.
get_target_property(CUTLASS_INCLUDES CUTLASS INTERFACE_INCLUDE_DIRECTORIES)
include_directories("${CUTLASS_INCLUDES}")
endif()
cutlass_add_executable(
00_basic_gemm
${EXAMPLES_BASIC_CUTLASS_GEMM_SOURCES}
)
examples/00_basic_gemm/basic_gemm.cu
+45 -47
View File
@@ -40,6 +40,11 @@
Aside from defining and launching the SGEMM kernel, this example does not use any other components
or utilities within CUTLASS. Such utilities are demonstrated elsewhere in other examples and are
prevalent in the CUTLASS unit tests.
This example has delibrately been kept similar to the basic_gemm example from cutass-1.3 to
highlight the minimum amount of differences needed to transition to cutlass-2.0.
Cutlass-1.3 sgemm: https://github.com/NVIDIA/cutlass/blob/master/examples/00_basic_gemm/basic_gemm.cu
*/
// Standard Library includes
@@ -47,17 +52,15 @@
#include <sstream>
#include <vector>
// Helper methods to check for errors
#include "helper.h"
//
// CUTLASS includes needed for single-precision GEMM kernel
//
// Defines cutlass::gemm::Gemm, the generic Gemm computation template class.
#include "cutlass/gemm/gemm.h"
// Defines cutlass::gemm::SgemmTraits, the structural components for single-precision GEMM
#include "cutlass/gemm/sgemm_traits.h"
#pragma warning( disable : 4503)
// Defines cutlass::gemm::device::Gemm, the generic Gemm computation template class.
#include "cutlass/gemm/device/gemm.h"
///////////////////////////////////////////////////////////////////////////////////////////////////
//
@@ -81,63 +84,58 @@ cudaError_t CutlassSgemmNN(
int ldc) {
// Define type definition for single-precision CUTLASS GEMM with column-major
// input matrices and 128x128x8 threadblock tile size.
//
// Note, GemmTraits<> is a generic template defined for various general matrix product
// computations within CUTLASS. It is intended to be maximally flexible, and consequently
// it contains numerous template arguments.
// input matrices and 128x128x8 threadblock tile size (chosen by default).
//
// To keep the interface manageable, several helpers are defined for plausible compositions
// including the following example for single-precision GEMM. Typical values are used as
// default template arguments. See `cutlass/gemm/gemm_traits.h` for more details.
// default template arguments. See `cutlass/gemm/device/default_gemm_configuration.h` for more details.
//
typedef cutlass::gemm::SgemmTraits<
cutlass::MatrixLayout::kColumnMajor, // layout of A matrix
cutlass::MatrixLayout::kColumnMajor, // layout of B matrix
cutlass::Shape<8, 128, 128> // threadblock tile size
>
GemmTraits;
// To view the full gemm device API interface, see `cutlass/gemm/device/gemm.h`
// Define a CUTLASS GEMM type from a GemmTraits<> instantiation.
typedef cutlass::gemm::Gemm<GemmTraits> Gemm;
using ColumnMajor = cutlass::layout::ColumnMajor;
// Construct and initialize CUTLASS GEMM parameters object.
using CutlassGemm = cutlass::gemm::device::Gemm<float, // Data-type of A matrix
ColumnMajor, // Layout of A matrix
float, // Data-type of B matrix
ColumnMajor, // Layout of B matrix
float, // Data-type of C matrix
ColumnMajor>; // Layout of C matrix
// Define a CUTLASS GEMM type
CutlassGemm gemm_operator;
// Construct the CUTLASS GEMM arguments object.
//
// One of CUTLASS's design patterns is to define parameters objects that are constructible
// One of CUTLASS's design patterns is to define gemm argument objects that are constructible
// in host code and passed to kernels by value. These may include pointers, strides, scalars,
// and other arguments needed by Gemm and its components.
//
// The benefits of this pattern are (1.) a structured, composable strategy for passing host-constructible
// arguments to kernels and (2.) minimized initialization overhead on kernel entry.
//
typename Gemm::Params params;
CutlassGemm::Arguments args({M , N, K}, // Gemm Problem dimensions
{A, lda}, // Tensor-ref for source matrix A
{B, ldb}, // Tensor-ref for source matrix B
{C, ldc}, // Tensor-ref for source matrix C
{C, ldc}, // Tensor-ref for destination matrix D (may be different memory than source C matrix)
{alpha, beta}); // Scalars used in the Epilogue
int result = params.initialize(
M, // GEMM M dimension
N, // GEMM N dimension
K, // GEMM K dimension
alpha, // scalar alpha
A, // matrix A operand
lda,
B, // matrix B operand
ldb,
beta, // scalar beta
C, // source matrix C
ldc,
C, // destination matrix C (may be different memory than source C matrix)
ldc
);
//
// Launch the CUTLASS GEMM kernel.
//
cutlass::Status status = gemm_operator(args);
if (result) {
std::cerr << "Failed to initialize CUTLASS Gemm::Params object." << std::endl;
return cudaErrorInvalidValue;
//
// Return a cudaError_t if the CUTLASS GEMM operator returned an error code.
//
if (status != cutlass::Status::kSuccess) {
return cudaErrorUnknown;
}
// Launch the CUTLASS GEMM kernel.
Gemm::launch(params);
// Return any errors associated with the launch or cudaSuccess if no error.
return cudaGetLastError();
// Return success, if no errors were encountered.
return cudaSuccess;
}
///////////////////////////////////////////////////////////////////////////////////////////////////
examples/02_cutlass_utilities/CMakeLists.txt → examples/01_cutlass_utilities/CMakeLists.txt
+3 -15
View File
@@ -20,19 +20,7 @@
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
set(EXAMPLES_CUTLASS_UTILITIES_SOURCES
cutlass_example_add_executable(
01_cutlass_utilities
cutlass_utilities.cu
)
if (NOT CUTLASS_NATIVE_CUDA)
# cuda_add_executable does not take interface include directories into account
# Let's fetch them and pass them to CUDA.
get_target_property(CUTLASS_INCLUDES CUTLASS INTERFACE_INCLUDE_DIRECTORIES)
include_directories("${CUTLASS_INCLUDES}")
endif()
cutlass_add_executable(
02_cutlass_utilities
${EXAMPLES_CUTLASS_UTILITIES_SOURCES}
)
)
examples/02_cutlass_utilities/cutlass_utilities.cu → examples/01_cutlass_utilities/cutlass_utilities.cu
+131 -120
View File
@@ -38,28 +38,28 @@
cutlass::half_t
This is a host-only implementation of a half-precision floating-point type. It requires no
specialized hardware support from the CPU and emulates arithmetic operations. Device-side code
should use CUDA's `half` type.
This is a numeric type implementing IEEE half-precision quantities. It is functional in host
and device code. In host-side code, CUTLASS_ENABLE_F16C optionally enables harware-accelerated
numeric conversion on x86-64 CPUs support F16C extensions. In device code, all available
hardware is used to implement conversion and numeric operations.
cutlass::HostMatrix<>
cutlass::HostTensor<>
This template class simplifies the creation of a rank=2 tensor with either a column-major or
row-major layout in memory.
This template class simplifies the creation of tensors for all supported layouts. It simplifies
allocation and management of host- and device- memory allocations.
This class offers methods device_view() and host_view() to provide TensorView objects for
device- and host-side memory allocations.
cutlass::reference::device::TensorInitialize()
cutlass::reference::device::TensorFillRandomGaussian()
This template function initializes the elements of a tensor according to either a procedural
definition or a random distribution. The function in namespace `cutlass::reference::device::`
uses a CUDA kernel to perform this initialization, relying on CURAND to compute random numbers.
This template function initializes elementsof a tensor to a random Gaussian distribution. It
uses cuRAND in device code to compute random numbers.
cutlass::reference::host::Gemm()
cutlass::reference::host::Gemm<>
This template function computes the general matrix product. This template supports unique
data types for each matrix operand, the internal accumulation type, and the scalar parameters
@@ -76,102 +76,82 @@
#include <iostream>
#include <sstream>
#include <vector>
#include <fstream>
#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__) >= 530
// CUTLASS includes needed for mixed-precision GEMM kernel
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/fp16_sgemm_traits.h"
// CUTLASS includes needed for half-precision GEMM kernel
#include "cutlass/cutlass.h"
#include "cutlass/core_io.h"
#include "cutlass/layout/matrix.h"
#include "cutlass/gemm/device/gemm.h"
//
// CUTLASS utility includes
//
// Defines operator<<() to write TensorView objects to std::ostream
#include "tools/util/tensor_view_io.h"
#include "cutlass/util/tensor_view_io.h"
// Defines cutlass::HostMatrix<>
#include "tools/util/host_matrix.h"
// Defines cutlass::HostTensor<>
#include "cutlass/util/host_tensor.h"
// Defines cutlass::half_t
#include "tools/util/half.h"
#include "cutlass/numeric_types.h"
// Defines cutlass::reference::device::TensorInitialize()
#include "tools/util/reference/device/tensor_elementwise.h"
// Defines device_memory::copy_device_to_device()
#include "cutlass/util/device_memory.h"
// Defines cutlass::reference::device::TensorFillRandomGaussian()
#include "cutlass/util/reference/device/tensor_fill.h"
// Defines cutlass::reference::host::TensorEquals()
#include "tools/util/reference/host/tensor_elementwise.h"
#include "cutlass/util/reference/host/tensor_compare.h"
// Defines cutlass::reference::host::Gemm()
#include "tools/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/gemm.h"
#pragma warning( disable : 4503)
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Define a CUTLASS GEMM template and launch a GEMM kernel.
cudaError_t Cutlass_FP16_SgemmNN(
cudaError_t cutlass_hgemm_nn(
int M,
int N,
int K,
cutlass::half_t alpha,
half const *A,
cutlass::half_t const *A,
int lda,
half const *B,
cutlass::half_t const *B,
int ldb,
cutlass::half_t beta,
half *C,
cutlass::half_t *C,
int ldc) {
// Define a CUTLASS Gemm using mixed-precision floating-point.
//
// A, B, C, D are half-precision. Internal accumulation is in single-precision.
//
// Note, we use CUDA's `half` type for device-side code including CUTLASS GEMM kernels.
//
typedef cutlass::gemm::Fp16SgemmSgemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<16, 128, 128>,
half, // A type
half, // B type
half, // C type
half, // D type
half // Scalar type: alpha, beta
>
GemmTraits;
// Define the GEMM operation
using Gemm = cutlass::gemm::device::Gemm<
cutlass::half_t, // ElementA
cutlass::layout::ColumnMajor, // LayoutA
cutlass::half_t, // ElementB
cutlass::layout::ColumnMajor, // LayoutB
cutlass::half_t, // ElementOutput
cutlass::layout::ColumnMajor // LayoutOutput
>;
// Define a CUTLASS GEMM object.
typedef cutlass::gemm::Gemm<GemmTraits> Gemm;
Gemm gemm_op;
cutlass::Status status = gemm_op({
{M, N, K},
{A, lda},
{B, ldb},
{C, ldc},
{C, ldc},
{alpha, beta}
});
// Construct and initialize CUTLASS GEMM parameters object.
typename Gemm::Params params;
int result = params.initialize(
M, // GEMM M dimension
N, // GEMM N dimension
K, // GEMM K dimension
reinterpret_cast<half const &>(alpha), // scalar alpha
A, // matrix A operand
lda,
B, // matrix B operand
ldb,
reinterpret_cast<half const &>(beta), // scalar beta
C, // source matrix C
ldc,
C, // destination matrix C (may be different memory than source C matrix)
ldc
);
if (result) {
std::cerr << "Failed to initialize CUTLASS Gemm::Params object." << std::endl;
return cudaErrorInvalidValue;
if (status != cutlass::Status::kSuccess) {
return cudaErrorUnknown;
}
// Launch the CUTLASS GEMM kernel.
Gemm::launch(params);
// Return any errors associated with the launch or cudaSuccess if no error.
return cudaGetLastError();
return cudaSuccess;
}
///////////////////////////////////////////////////////////////////////////////////////////////////
@@ -182,53 +162,70 @@ cudaError_t TestCutlassGemm(int M, int N, int K, cutlass::half_t alpha, cutlass:
cudaError_t result;
//
// Construct cutlass::HostMatrix<> using the half-precision host-side type.
// Construct cutlass::HostTensor<> using the half-precision host-side type.
//
// cutlass::HostMatrix<> allocates memory on both the host and device corresponding to rank=2
// cutlass::HostTensor<> allocates memory on both the host and device corresponding to rank=2
// tensors in column-major layout. Explicit synchronization methods are offered to copy the
// tensor to the device or to the host.
//
// M-by-K matrix of cutlass::half_t
cutlass::HostMatrix<cutlass::half_t> A(cutlass::MatrixCoord(M, K));
cutlass::HostTensor<cutlass::half_t, cutlass::layout::ColumnMajor> A(cutlass::MatrixCoord(M, K));
// K-by-N matrix of cutlass::half_t
cutlass::HostMatrix<cutlass::half_t> B(cutlass::MatrixCoord(K, N));
cutlass::HostTensor<cutlass::half_t, cutlass::layout::ColumnMajor> B(cutlass::MatrixCoord(K, N));
// M-by-N matrix of cutlass::half_t
cutlass::HostMatrix<cutlass::half_t> C_cutlass(cutlass::MatrixCoord(M, N));
cutlass::HostTensor<cutlass::half_t, cutlass::layout::ColumnMajor> C_cutlass(cutlass::MatrixCoord(M, N));
// M-by-N matrix of cutlass::half_t
cutlass::HostMatrix<cutlass::half_t> C_reference(cutlass::MatrixCoord(M, N));
cutlass::HostTensor<cutlass::half_t, cutlass::layout::ColumnMajor> C_reference(cutlass::MatrixCoord(M, N));
//
// Initialize matrices with small, random integers.
//
cutlass::Distribution dist;
// Uniform random distribution from -4 .. 4. Values are truncated to integers.
dist.set_uniform(-4, 4);
// Arbitrary RNG seed value. Hard-coded for deterministic results.
int seed = 2080;
uint64_t seed = 2080;
cutlass::reference::device::TensorInitialize(
A.device_view(), // concept: TensorView
// Gaussian random distribution
cutlass::half_t mean = 0.0_hf;
cutlass::half_t stddev = 5.0_hf;
// Specify the number of bits right of the binary decimal that are permitted
// to be non-zero. A value of "0" here truncates random values to integers
int bits_less_than_one = 0;
cutlass::reference::device::TensorFillRandomGaussian(
A.device_view(),
seed,
dist);
mean,
stddev,
bits_less_than_one
);
cutlass::reference::device::TensorFillRandomGaussian(
B.device_view(),
seed * 2019,
mean,
stddev,
bits_less_than_one
);
cutlass::reference::device::TensorFillRandomGaussian(
C_cutlass.device_view(),
seed * 1993,
mean,
stddev,
bits_less_than_one
);
cutlass::reference::device::TensorInitialize(
B.device_view(), // concept: TensorView
seed * 2,
dist);
cutlass::reference::device::TensorInitialize(
C_cutlass.device_view(), // concept: TensorView
seed * 3,
dist);
// Copy C_cutlass into C_reference so the GEMM is correct when beta != 0.
cutlass::reference::device::TensorFill(C_reference.device_view(), C_cutlass.device_view());
cutlass::device_memory::copy_device_to_device(
C_reference.device_data(),
C_cutlass.device_data(),
C_cutlass.capacity());
// Copy the device-side view into host memory
C_reference.sync_host();
@@ -237,18 +234,18 @@ cudaError_t TestCutlassGemm(int M, int N, int K, cutlass::half_t alpha, cutlass:
// Launch the CUTLASS GEMM kernel
//
result = Cutlass_FP16_SgemmNN(
result = cutlass_hgemm_nn(
M,
N,
K,
alpha,
A.device_data(),
A.leading_dim(),
A.stride(0),
B.device_data(),
B.leading_dim(),
B.stride(0),
beta,
C_cutlass.device_data(),
C_cutlass.leading_dim()
C_cutlass.stride(0)
);
if (result != cudaSuccess) {
@@ -268,20 +265,34 @@ cudaError_t TestCutlassGemm(int M, int N, int K, cutlass::half_t alpha, cutlass:
C_cutlass.sync_host();
// Compute the reference result using the host-side GEMM reference implementation.
cutlass::reference::host::Gemm(
cutlass::gemm::GemmCoord(K, N, M), // problem size (type: cutlass::gemm::GemmCoord)
alpha, // alpha (type: cutlass::half_t)
A.host_ref(), // A (concept: TensorRef)
B.host_ref(), // B (concept: TensorRef)
beta, // beta (type: cutlass::half_t)
C_reference.host_ref(), // C (concept: TensorRef)
float(0) // Accumulator initial value passed as argument to deduce
); // internal accumulation data type as float.
cutlass::reference::host::Gemm<
cutlass::half_t, // ElementA
cutlass::layout::ColumnMajor, // LayoutA
cutlass::half_t, // ElementB
cutlass::layout::ColumnMajor, // LayoutB
cutlass::half_t, // ElementOutput
cutlass::layout::ColumnMajor, // LayoutOutput
cutlass::half_t,
cutlass::half_t
> gemm_ref;
gemm_ref(
{M, N, K}, // problem size (type: cutlass::gemm::GemmCoord)
alpha, // alpha (type: cutlass::half_t)
A.host_ref(), // A (type: TensorRef<half_t, ColumnMajor>)
B.host_ref(), // B (type: TensorRef<half_t, ColumnMajor>)
beta, // beta (type: cutlass::half_t)
C_reference.host_ref() // C (type: TensorRef<half_t, ColumnMajor>)
);
// Compare reference to computed results.
if (!cutlass::reference::host::TensorEquals(C_reference.host_view(), C_cutlass.host_view())) {
if (!cutlass::reference::host::TensorEquals(
C_reference.host_view(),
C_cutlass.host_view())) {
std::cerr << "Error - CUTLASS mixed-precision GEMM kernel differs from reference." << std::endl;
char const *filename = "errors_01_cutlass_utilities.csv";
std::cerr << "Error - CUTLASS GEMM kernel differs from reference. Wrote computed and reference results to '" << filename << "'" << std::endl;
//
// On error, print C_cutlass and C_reference to std::cerr.
@@ -290,11 +301,13 @@ cudaError_t TestCutlassGemm(int M, int N, int K, cutlass::half_t alpha, cutlass:
// arrays of type cutlass::half_t.
//
// Result of CUTLASS mixed-precision GEMM kernel
std::cerr << "CUTLASS:\n" << C_cutlass << std::endl;
std::ofstream file(filename);
// Result of CUTLASS GEMM kernel
file << "\n\nCUTLASS =\n" << C_cutlass.host_view() << std::endl;
// Result of reference computation
std::cerr << "Reference:\n" << C_reference << std::endl;
file << "\n\nReference =\n" << C_reference.host_view() << std::endl;
// Return error code.
return cudaErrorUnknown;
@@ -327,7 +340,7 @@ int main(int argc, const char *arg[]) {
}
if (!(prop.major > 5 || (prop.major == 5 && prop.minor >= 3))) {
std::cerr << "This example uses mixed precision and is only suitable for devices with compute capability 5.3 or greater.\n";
std::cerr << "This example uses half precision and is only suitable for devices with compute capability 5.3 or greater.\n";
std::cerr << "You are using a CUDA device with compute capability " << prop.major << "." << prop.minor << std::endl;
return -1;
}
@@ -349,7 +362,7 @@ int main(int argc, const char *arg[]) {
//
// Values outside the range of IEEE FP16 will overflow to infinity or underflow to zero.
//
cutlass::half_t scalars[2] = { 1, 0 };
cutlass::half_t scalars[2] = { 1.0_hf, 0.0_hf };
for (int i = 4; i < argc && i < 6; ++i) {
std::stringstream ss(arg[i]);
@@ -379,5 +392,3 @@ int main(int argc, const char *arg[]) {
///////////////////////////////////////////////////////////////////////////////////////////////////
#endif
examples/01_tensor_view/tensor_view.cu
-424
View File
@@ -1,424 +0,0 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*
This example demonstrates operations using TensorRef<> and TensorView<> as well as their explicit
equivalent functionality in CUDA code.
CUTLASS provides abstractions for interacting with multidimension tensors in device memory.
Consequently, we define a hierarchy of pointer-like types for referencing tensors.
T * - raw pointer to elements of type T
cutlass::TensorRef<T, Rank> - reference to a tensor of elements of type T and given rank.
Includes a mapping function and associated stride vector for
accessing elements in linear memory.
cutlass::TensorView<T, Rank>: - extends TensorRef<> by adding bounds information. This is a
public TensorRef<T, Rank> complete mathematical object which may be used as the argument
to CUTLASS functions.
The above provide an identity maping of a logical index space to linear memory. An element
at logical coordinate X has an offset computed as follows:
offset = dot(X, stride)
where dot() computes the inner product of X and a vector of "strides."
CUTLASS 1.1 introduces a mapping function and an additional 'rank' to offer a flexible way to
map the logical index space of the tensor to memory. The mapping function maps a coordinate
of rank R to an index space of rank S. The linear offset is computed as:
offset = dot( MapFunc(X), stride )
where stride is a vector of rank S.
The complete template declaration for cutlass::TensorRef<> is as follows.
template <
/// Data type of element stored within tensor
typename Storage,
/// Rank of logical tensor
int Rank,
/// Maps a Coord<Rank> in the logical tensor index space to the internal n-D array
typename MapFunc = IdentityTensorMapFunc<Rank>,
/// Rank of internal n-D array
int StorageRank_ = MapFunc::kStorageRank,
/// Index type used for coordinates
typename Index = int,
/// Index type used for offsets and pointer differences
typename LongIndex = long long
>
class TensorRef;
CUTLASS kernels make extensive use of vectorization of memory accesses for efficiency and
correctness. Consequently, we enforce a constraint on the strides used by mapping functions
such that:
1. The "fastest-changing" stride is always 1 thereby mandating that consecutive elements in
that rank are consecutive in linear memory.
2. The fastest changing rank is always last in the stride vector and not explicitly stored.
Thus, the stride vector used by mapping functions has length of one fewer than the rank of the
storage tensor. These constraints are consistent with the BLAS interface of passing matrices as
a tuple consisting of a pointer and a "leading dimension." In fact, these are rank=2 tensors
whose fastest changing dimension is 1, and the stride vector is of length 1.
A typical mapping function might simply map the rows and columns of a matrix, a rank=2 tensor,
to linear memory such that (1.) elements in the same column are consecutive in memory
(column-major), or (2.) elements in the same row are consecutive (row-major). These can be
accomplished by two different mapping functions whose stride vector is length=2. The first
element is the "leading dimension."
The following mapping functions demonstrates mappings for these canonical matrix layouts. In
both cases, the logical index space is referenced by coordinates of the form (row, column).
// cutlass/matrix_traits.h
struct MatrixLayout {
//
// TensorRefMapFunc definitions for common layouts
//
/// Mapping function for row-major matrices
struct RowMajor {
/// Storage rank = 2 implies stride vector: (ldm, 1)
static int const kStorageRank = 2;
/// Maps (row, col) to (row, col)
CUTLASS_HOST_DEVICE
Coord<kStorageRank> operator()(Coord<2> const &coord) const {
return coord;
}
};
/// Mapping function for column-major matrices
struct ColumnMajor {
/// Storage rank = 2 implies stride vector: (ldm, 1)
static int const kStorageRank = 2;
/// Maps (row, col) to (col, row)
CUTLASS_HOST_DEVICE
Coord<kStorageRank> operator()(Coord<2> const &coord) const {
return make_Coord(coord[1], coord[0]);
}
};
};
The requirement that the fastest-changing stride always be of unit size need not be a limitation.
To implement "sparse" computations or matrix operations in which matrix elements have arbitrary
stride along both row and column, define a mapping function whose storage rank is 3. This permits
two elements of the stride vector to have a non-unit value. The map function defined in
`cutlass::MatrixTraits::ContiguousLayout` is an example.
```
/// Mapping function for scenario in which layout is row-major or column-major but this information
/// is only available at runtime.
struct ContiguousLayout {
/// Arbitrary storage rank
static int const kStorageRank = 3;
/// Dimension of rows
static int const kRow = 0;
/// Dimension of columns
static int const kColumn = 1;
/// Mapping function defined by runtime variable. Returns coordinates in n-D storage array
/// as (matrix row, matrix colum, 0)
CUTLASS_HOST_DEVICE
Coord<kStorageRank> operator()(MatrixCoord const &coord) const {
return make_Coord(coord.row(), coord.column(), 0);
}
/// Helper to construct a stride vector based on contiguous matrix layout and leading dimension
CUTLASS_HOST_DEVICE
static Coord<kStorageRank> stride(MatrixLayout::Kind layout, int ldm) {
if (layout == MatrixLayout::kRowMajor) {
return make_Coord(ldm, 1, 1);
}
return make_Coord(1, ldm, 1);
}
};
```
cutlass::TensorView<> extends this concept by including a size vector to specify the bounds of
the index space. The value of each coordinate in the size vector defines the half-open range of
indices whose smallest value is zero.
*/
///////////////////////////////////////////////////////////////////////////////////////////////////
// Standard Library includes
#include <iostream>
#include <vector>
//
// CUTLASS includes
//
// Defines cutlass::Coord<>
#include "cutlass/coord.h"
// Defines cutlass::TensorRef<>
#include "cutlass/tensor_ref.h"
// Defines cutlass::TensorView<>
#include "cutlass/tensor_view.h"
// Defines cutlass::MatrixLayout
#include "cutlass/matrix_traits.h"
///////////////////////////////////////////////////////////////////////////////////////////////////
//
// Column-major matrix access
//
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Define a rank=2 tensor modeling a column-major matrix
typedef cutlass::TensorView<
int, // storage element is of type int
2, // tensor has rank=2 logical index space
cutlass::MatrixLayout::ColumnMajor // column-major mapping function
> TensorViewColumnMajor;
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Kernel to copy a matrix from raw memory into a cutlass::TensorView
__global__ void MatrixCopyColumnMajor(
TensorViewColumnMajor destination, // destination tensor accessed by TensorView
int const *source, // source matrix accessed using cuBLAS-style pointer
int ldm) { // and leading dimension
// Compute unique row and column for each thread
int row = threadIdx.x + blockIdx.x * blockDim.x;
int column = threadIdx.y + blockIdx.y * blockDim.y;
// Define a coordinate based on the thread's row and column
cutlass::Coord<2> coord = cutlass::make_Coord(row, column);
// Bounds test
if (coord < destination.size()) {
// Access the element
destination.at(coord) = source[row + column * ldm];
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Launches kernel MatrixCopyColumnMajor()
cudaError_t TestMatrixCopyColumnMajor() {
cudaError_t result;
int const M = 32; // number of rows
int const N = 16; // number of columns
int const ldm = 40; // matrix leading dimension
//
// Allocate source and destination matrices
//
int *Destination;
int *Source;
int const matrix_capacity = ldm * N; // number of elements in memory needed to store matrix
size_t const sizeof_matrix = sizeof(int) * matrix_capacity; // size of matrix in bytes
// Allocate destination and source matrices
result = cudaMalloc((void **)&Destination, sizeof_matrix);
if (result != cudaSuccess) {
std::cerr << "Failed to allocate destination matrix on device: " << cudaGetErrorString(result) << std::endl;
return result;
}
result = cudaMalloc((void **)&Source, sizeof_matrix);
if (result != cudaSuccess) {
cudaFree(Destination);
std::cerr << "Failed to allocate source matrix on device:" << cudaGetErrorString(result) << std::endl;
return result;
}
// Clear destination matrix in device memory
result = cudaMemset(Destination, 0, sizeof_matrix);
if (result != cudaSuccess) {
cudaFree(Destination);
cudaFree(Source);
std::cerr << "Failed to clear destination matrix: " << cudaGetErrorString(result) << std::endl;
return result;
}
//
// Initialize matrix
//
std::vector<int> source_host(matrix_capacity, 0);
// Procedurally generate input results using several arbitrary constants.
int const magic_row_stride = 2;
int const magic_column_stride = 3;
for (int j = 0; j < N; ++j) {
for (int i = 0; i < M; ++i) {
source_host.at(i + j * ldm) = i * magic_row_stride + j * magic_column_stride;
}
}
// Copy to device memory
result = cudaMemcpy(Source, source_host.data(), sizeof_matrix, cudaMemcpyHostToDevice);
if (result != cudaSuccess) {
cudaFree(Destination);
cudaFree(Source);
std::cerr << "Failed to copy from host to source matrix: " << cudaGetErrorString(result) << std::endl;
return result;
}
//
// Define a TensorView<> pointing to the destination matrix
//
TensorViewColumnMajor destination_view_device(
Destination, // pointer to base of matrix in device memory
cutlass::make_Coord(ldm, 1), // stride vector
cutlass::make_Coord(M, N) // bounds of matrix
);
//
// Launch kernel to copy matrix
//
dim3 block(16, 16);
dim3 grid((M + block.x - 1) / block.x, (N + block.y - 1) / block.y);
MatrixCopyColumnMajor<<< grid, block >>>(destination_view_device, Source, ldm);
result = cudaGetLastError();
if (result != cudaSuccess) {
std::cerr << "Kernel MatrixCopyColumnMajor() failed: "
<< cudaGetErrorString(result) << std::endl;
cudaFree(Destination);
cudaFree(Source);
return result;
}
//
// Copy results to host memory
//
std::vector<int> dest_host(matrix_capacity, 0);
result = cudaMemcpy(dest_host.data(), Destination, sizeof_matrix, cudaMemcpyDeviceToHost);
if (result != cudaSuccess) {
std::cerr << "Failed to copy destination matrix to host memory: "
<< cudaGetErrorString(result) << std::endl;
cudaFree(Destination);
cudaFree(Source);
return result;
}
//
// Verify result
//
// Define a TensorView for use in accessing host memory
TensorViewColumnMajor destination_view_host(
dest_host.data(), // pointer to base of matrix in host memory
cutlass::make_Coord(ldm, 1), // stride vector
cutlass::make_Coord(M, N) // bounds of matrix
);
// Verify against procedurally computed results
for (int j = 0; j < N; ++j) {
for (int i = 0; i < M; ++i) {
// computed result
int expected = i * magic_row_stride + j * magic_column_stride;
// access data by computing explicit offsets
int got_explicit = dest_host.at(i + j * ldm);
// access data in host memory through a TensorView
int got_view = destination_view_host.at(cutlass::make_Coord(i, j));
if (got_explicit != expected) {
std::cerr << "Error at element (" << i << ", " << j
<< ") accessed through explicitly computed offset - expected: " << expected
<< ", got: " << got_explicit << std::endl;
return cudaErrorUnknown;
}
if (got_view != expected) {
std::cerr << "Error at element (" << i << ", " << j
<< ") accesed through TensorView<> on the host - expected: " << expected
<< ", got: " << got_view << std::endl;
return cudaErrorUnknown;
}
}
}
return cudaSuccess;
}
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Entry point for tensor_view example.
//
// usage:
//
// 02_tensor_view
//
int main() {
cudaError_t result = TestMatrixCopyColumnMajor();
if (result == cudaSuccess) {
std::cout << "Passed" << std::endl;
}
return (result == cudaSuccess ? 0 : -1);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
examples/05_wmma_gemm/CMakeLists.txt → examples/02_dump_reg_shmem/CMakeLists.txt
+3 -15
View File
@@ -20,19 +20,7 @@
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
set(EXAMPLES_BASIC_CUTLASS_GEMM_SOURCES
wmma_gemm.cu
)
if (NOT CUTLASS_NATIVE_CUDA)
# cuda_add_executable does not take interface include directories into account
# Let's fetch them and pass them to CUDA.
get_target_property(CUTLASS_INCLUDES CUTLASS INTERFACE_INCLUDE_DIRECTORIES)
include_directories("${CUTLASS_INCLUDES}")
endif()
cutlass_add_executable(
05_wmma_gemm
${EXAMPLES_BASIC_CUTLASS_GEMM_SOURCES}
cutlass_example_add_executable(
02_dump_reg_shmem
dump_reg_shmem.cu
)
examples/02_dump_reg_shmem/dump_reg_shmem.cu
+179
View File
@@ -0,0 +1,179 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
*modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice,
*this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
*notice, this list of conditions and the following disclaimer in the
*documentation and/or other materials provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its
*contributors may be used to endorse or promote products derived from this
*software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
*AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
*IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
*DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY DIRECT,
*INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
*DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
*OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TOR (INCLUDING
*NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
*EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Demonstrate CUTLASS debugging tool for dumping fragments and shared
memory
*/
///////////////////////////////////////////////////////////////////////////////////////////////////
// Standard Library includes
#include <iostream>
//
// CUTLASS includes
//
#include "cutlass/aligned_buffer.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/layout/matrix.h"
#include "cutlass/matrix_shape.h"
#include "cutlass/numeric_types.h"
#include "cutlass/core_io.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/tensor_view_io.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/transform/pitch_linear_thread_map.h"
#include "cutlass/transform/threadblock/predicated_tile_iterator.h"
#include "cutlass/transform/threadblock/regular_tile_iterator_tensor_op.h"
#include "cutlass/util/debug.h"
#include "cutlass/util/device_dump.h"
#define EXAMPLE_MATRIX_ROW 64
#define EXAMPLE_MATRIX_COL 32
///////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Element, typename GmemIterator, typename SmemIterator>
__global__ void kernel_dump(typename GmemIterator::Params params,
typename GmemIterator::TensorRef ref) {
__shared__ Element shared_storage[EXAMPLE_MATRIX_ROW * EXAMPLE_MATRIX_COL];
// Construct the global iterator and load the data to the fragments.
int tb_thread_id = threadIdx.y * blockDim.x + threadIdx.x;
GmemIterator gmem_iterator(params, ref.data(),
{EXAMPLE_MATRIX_ROW, EXAMPLE_MATRIX_COL},
tb_thread_id);
typename GmemIterator::Fragment frag;
frag.clear();
gmem_iterator.load(frag);
// Call dump_fragment() with different parameters.
if (threadIdx.x == 0 && blockIdx.x == 0)
printf("\nAll threads dump all the elements:\n");
cutlass::debug::dump_fragment(frag);
if (threadIdx.x == 0 && blockIdx.x == 0)
printf("\nFirst thread dumps all the elements:\n");
cutlass::debug::dump_fragment(frag, /*N = */ 1);
if (threadIdx.x == 0 && blockIdx.x == 0)
printf("\nFirst thread dumps first 16 elements:\n");
cutlass::debug::dump_fragment(frag, /*N = */ 1, /*M = */ 16);
if (threadIdx.x == 0 && blockIdx.x == 0)
printf("\nFirst thread dumps first 16 elements with a stride of 8:\n");
cutlass::debug::dump_fragment(frag, /*N = */ 1, /*M = */ 16, /*S = */ 8);
// Construct the shared iterator and store the data to the shared memory.
SmemIterator smem_iterator(
typename SmemIterator::TensorRef(
{shared_storage, SmemIterator::Layout::packed(
{EXAMPLE_MATRIX_ROW, EXAMPLE_MATRIX_COL})}),
tb_thread_id);
smem_iterator.store(frag);
// Call dump_shmem() with different parameters.
if (threadIdx.x == 0 && blockIdx.x == 0) printf("\nDump all the elements:\n");
cutlass::debug::dump_shmem(shared_storage,
EXAMPLE_MATRIX_ROW * EXAMPLE_MATRIX_COL);
if (threadIdx.x == 0 && blockIdx.x == 0)
printf("\nDump all the elements with a stride of 8:\n");
cutlass::debug::dump_shmem(
shared_storage, EXAMPLE_MATRIX_ROW * EXAMPLE_MATRIX_COL, /*S = */ 8);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Entry point for dump_reg_shmem example.
//
// usage:
//
// 02_dump_reg_shmem
//
int main() {
// Initialize a 64x32 column major matrix with sequential data (1,2,3...).
using Element = cutlass::half_t;
using Layout = cutlass::layout::ColumnMajor;
cutlass::HostTensor<Element, Layout> matrix(
{EXAMPLE_MATRIX_ROW, EXAMPLE_MATRIX_COL});
cutlass::reference::host::BlockFillSequential(matrix.host_data(),
matrix.capacity());
// Dump the matrix.
std::cout << "Matrix:\n" << matrix.host_view() << "\n";
// Copy the matrix to the device.
matrix.sync_device();
// Define a global iterator, a shared iterator and their thread map.
using ThreadMap = cutlass::transform::PitchLinearWarpRakedThreadMap<
cutlass::layout::PitchLinearShape<EXAMPLE_MATRIX_ROW, EXAMPLE_MATRIX_COL>,
32, cutlass::layout::PitchLinearShape<8, 4>, 8>;
using GmemIterator =
cutlass::transform::threadblock::PredicatedTileIterator<
cutlass::MatrixShape<EXAMPLE_MATRIX_ROW, EXAMPLE_MATRIX_COL>, Element,
Layout, 1, ThreadMap>;
typename GmemIterator::Params params(matrix.layout());
using SmemIterator = cutlass::transform::threadblock::RegularTileIterator<
cutlass::MatrixShape<EXAMPLE_MATRIX_ROW, EXAMPLE_MATRIX_COL>, Element,
cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<16, 64>, 1,
ThreadMap>;
dim3 grid(1, 1);
dim3 block(32, 1, 1);
kernel_dump<Element, GmemIterator, SmemIterator>
<<<grid, block>>>(params, matrix.device_ref());
cudaError_t result = cudaDeviceSynchronize();
if (result != cudaSuccess) {
std::cout << "Failed" << std::endl;
}
return (result == cudaSuccess ? 0 : -1);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
examples/01_tensor_view/CMakeLists.txt → examples/03_visualize_layout/CMakeLists.txt
+11 -15
View File
@@ -20,19 +20,15 @@
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
set(EXAMPLES_TENSOR_VIEW_SOURCES
tensor_view.cu
)
if (NOT CUTLASS_NATIVE_CUDA)
# cuda_add_executable does not take interface include directories into account
# Let's fetch them and pass them to CUDA.
get_target_property(CUTLASS_INCLUDES CUTLASS INTERFACE_INCLUDE_DIRECTORIES)
include_directories("${CUTLASS_INCLUDES}")
endif()
cutlass_add_executable(
01_tensor_view
${EXAMPLES_TENSOR_VIEW_SOURCES}
)
03_visualize_layout
visualize_layout.cpp
register_layout.cu
)
target_link_libraries(
03_visualize_layout
PRIVATE
CUTLASS
cutlass_tools_util_includes
)
examples/03_visualize_layout/options.h
+115
View File
@@ -0,0 +1,115 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#include <vector>
#include <iostream>
// Cutlass command line parser
#include "cutlass/util/command_line.h"
class Options {
public:
bool help;
bool good;
std::vector<int> extent; ///< extent of tile to fill
std::vector<int> stride; ///< stride vector for layout function
std::vector<int> output_shape; ///< output shape
int vectorize; ///< sequences of consecutive output elements are concatenated into a vector
/// if, and only if, they were consecutive in source memory
public:
/// Options
Options():
help(false),
good(true),
extent({32, 8}),
stride({32}),
output_shape({16, 8}),
vectorize(1) {
}
/// Constructs from command line parser
Options(cutlass::CommandLine const & cmd_line): help(false), good(true) {
if (cmd_line.check_cmd_line_flag("help") ||
cmd_line.check_cmd_line_flag("h")) {
help = true;
}
if (cmd_line.check_cmd_line_flag("extent")) {
cmd_line.get_cmd_line_arguments("extent", extent);
}
else {
extent = {32, 8};
}
if (cmd_line.check_cmd_line_flag("stride")) {
cmd_line.get_cmd_line_arguments("stride", stride);
}
int default_output_shape[] = {16, 8};
if (cmd_line.check_cmd_line_flag("output-shape")) {
cmd_line.get_cmd_line_arguments("output-shape", output_shape);
}
for (int i = int(output_shape.size()); i < 2; ++i) {
output_shape.push_back(default_output_shape[i]);
}
if (cmd_line.check_cmd_line_flag("vectorize")) {
cmd_line.get_cmd_line_argument("vectorize", vectorize);
}
else {
vectorize = 1;
}
if (output_shape.front() % vectorize) {
std::cerr << "Error: --vectorize=" << vectorize
<< " must divide contiguous elements in --output-shape="
<< output_shape.at(0) << "," << output_shape.at(1) << std::endl;
good = false;
}
}
/// Prints usage statement
static void print_usage(std::ostream &out) {
out
<< " Options:\n"
<< " --help Displays this help message.\n"
<< " --extent=<extent> Specifies the layout-specific extent (as comma-delimited array).\n"
<< " --stride=<stride> Specifies the layout-specific stride vector (comma-delimited array)\n"
<< " --output-shape=<extent> Specifies the dimensions of a row-major output matrix. \n"
<< " --vectorize=<vector length> If possible, vectorizes the output into vectors of consecutive elements\n";
}
};
examples/03_visualize_layout/register_layout.cu
+93
View File
@@ -0,0 +1,93 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief CUTLASS layout visualization example
*/
#include <map>
#include <memory>
#include "cutlass/layout/matrix.h"
#include "cutlass/layout/pitch_linear.h"
#include "cutlass/layout/tensor_op_multiplicand_sm70.h"
#include "cutlass/layout/tensor_op_multiplicand_sm75.h"
#include "visualize_layout.h"
#include "register_layout.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
void RegisterLayouts(std::map<std::string, std::unique_ptr<VisualizeLayoutBase> > &layouts) {
struct {
char const *name;
VisualizeLayoutBase *ptr;
} layout_pairs[] = {
{"PitchLinear", new VisualizeLayout<cutlass::layout::PitchLinear>},
{"ColumnMajor", new VisualizeLayout<cutlass::layout::ColumnMajor>},
{"RowMajor", new VisualizeLayout<cutlass::layout::RowMajor>},
{"ColumnMajorInterleaved<4>",
new VisualizeLayout<cutlass::layout::ColumnMajorInterleaved<4>>},
{"RowMajorInterleaved<4>",
new VisualizeLayout<cutlass::layout::RowMajorInterleaved<4>>},
// Integer matrix multiply.int4 8832 Interleaved-64
{"TensorOpMultiplicand<4,64>",
new VisualizeLayout<cutlass::layout::TensorOpMultiplicand<4, 64>>},
// Integer matrix multiply.int4 8832 TN kblock128
{"TensorOpMultiplicand<4,128>",
new VisualizeLayout<cutlass::layout::TensorOpMultiplicand<4, 128>>},
// Integer matrix multiply 8816 Interleaved-32
{"TensorOpMultiplicand<8,32>",
new VisualizeLayout<cutlass::layout::TensorOpMultiplicand<8, 32>>},
// Integer matrix multiply 8816 TN kblock64
{"TensorOpMultiplicand<8,64>",
new VisualizeLayout<cutlass::layout::TensorOpMultiplicand<8, 64>>},
// Matrix Multiply 1688 TN kblock32
{"TensorOpMultiplicand<16,32>",
new VisualizeLayout<cutlass::layout::TensorOpMultiplicand<16, 32>>},
// Matrix multiply 1688 NT
{"TensorOpMultiplicand<16,64>",
new VisualizeLayout<cutlass::layout::TensorOpMultiplicand<16, 64>>},
{"TensorOpMultiplicandCongruous<128,4>",
new VisualizeLayout<
cutlass::layout::TensorOpMultiplicandCongruous<128, 4>>},
{"TensorOpMultiplicandCrosswise<128,4>",
new VisualizeLayout<
cutlass::layout::TensorOpMultiplicandCrosswise<128, 4>>},
{"VoltaTensorOpMultiplicandCongruous<16>",
new VisualizeLayout<
cutlass::layout::VoltaTensorOpMultiplicandCongruous<16>>},
{"VoltaTensorOpMultiplicandCrosswise<16,32>",
new VisualizeLayout<
cutlass::layout::VoltaTensorOpMultiplicandCrosswise<16, 32>>},
};
for (auto layout : layout_pairs) {
layouts.emplace(std::string(layout.name), std::unique_ptr<VisualizeLayoutBase>(layout.ptr));
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////
examples/03_visualize_layout/register_layout.h
+53
View File
@@ -0,0 +1,53 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief CUTLASS layout visualization example
*/
#pragma once
#include <map>
#include <memory>
#include "options.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
struct VisualizeLayoutBase {
virtual bool visualize(Options const &) = 0;
virtual bool verify(bool verbose, std::ostream &out) = 0;
virtual void print_csv(std::ostream &out, char delim = '|', char new_line = '\n') = 0;
virtual std::ostream &print_help(std::ostream &out) {
return out;
}
virtual ~VisualizeLayoutBase() { }
};
/////////////////////////////////////////////////////////////////////////////////////////////////
void RegisterLayouts(std::map<std::string, std::unique_ptr<VisualizeLayoutBase> > &layouts);
/////////////////////////////////////////////////////////////////////////////////////////////////
examples/03_visualize_layout/visualize_layout.cpp
+127
View File
@@ -0,0 +1,127 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief CUTLASS layout visualization tool
*/
#include <map>
#include <iostream>
#include <iomanip>
#include <memory>
#include "options.h"
#include "register_layout.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
std::map<std::string, std::unique_ptr<VisualizeLayoutBase> > layouts;
/////////////////////////////////////////////////////////////////////////////////////////////////
void print_usage(std::ostream &out) {
out << "03_visualize_layout <layout> [options]"
<< "\n\n"
<< " Layouts:\n";
for (auto const & layout : layouts) {
out << " " << layout.first << std::string(46 - layout.first.size(), ' ');
layout.second->print_help(out);
out << "\n";
}
out << "\n";
Options::print_usage(out);
out << "\nExamples:\n\n"
<< "$ 03_visualize_layout RowMajor --extent=16,16\n"
<< "$ 03_visualize_layout \"ColumnMajorInterleaved<4>\" --extent=32,8 "
"--output-shape=16 --vectorize=4\n"
<< "$ 03_visualize_layout \"TensorOpMultiplicand<4,64>\" "
"--extent=64,64 --vectorize=32 --output-shape=256,4\n"
<< "$ 03_visualize_layout \"TensorOpMultiplicand<4,128>\" "
"--extent=128,32 --vectorize=32 --output-shape=256,4\n"
<< "$ 03_visualize_layout \"TensorOpMultiplicand<8,32>\" "
"--extent=32,64 --vectorize=16 --output-shape=128,4\n"
<< "$ 03_visualize_layout \"TensorOpMultiplicand<8,64>\" "
"--extent=64,32 --vectorize=16 --output-shape=128,4\n"
<< "$ 03_visualize_layout \"TensorOpMultiplicand<16,32>\" "
"--extent=32,32 --vectorize=8 --output-shape=64,4\n"
<< "$ 03_visualize_layout \"TensorOpMultiplicand<16,64>\" "
"--extent=64,16 --vectorize=8 --output-shape=64,4\n"
<< "$ 03_visualize_layout \"VoltaTensorOpMultiplicandCrosswise<16,32>\" "
"--extent=32,64 --vectorize=4 --output-shape=64,4\n"
<< "$ 03_visualize_layout \"VotlaTensorOpMultiplicandCongruous<16>\" "
"--extent=64,32 --vectorize=8 --output-shape=64,4\n";
out << std::endl;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Entry point
int main(int argc, char const *arg[]) {
RegisterLayouts(layouts);
if (argc == 1 || (std::string(arg[0]) == "-h" || std::string(arg[1]) == "--help")) {
print_usage(std::cout);
return 0;
}
// parse command line, skipping layout name
cutlass::CommandLine cmd_line(argc - 1, arg + 1);
Options options(cmd_line);
if (options.help) {
print_usage(std::cout);
return 0;
}
if (!options.good) {
return -1;
}
std::string layout_name = arg[1];
auto layout_it = layouts.find(layout_name);
if (layout_it == layouts.end()) {
std::cerr << "Layout '" << layout_name << "' not supported." << std::endl;
return -1;
}
bool passed = layout_it->second->visualize(options);
if (!passed) {
return -1;
}
layout_it->second->print_csv(std::cout);
return 0;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
examples/03_visualize_layout/visualize_layout.h
+377
View File
@@ -0,0 +1,377 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief CUTLASS layout visualization example
*/
#pragma once
#include <algorithm>
#include <stdexcept>
#include <vector>
#include "cutlass/coord.h"
#include "cutlass/util/reference/host/tensor_foreach.h"
#include "register_layout.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Permits copying dynamic vectors into static-length vectors
template <typename TensorCoord, int Rank>
struct vector_to_coord {
vector_to_coord(TensorCoord &coord, std::vector<int> const &vec) {
coord[Rank - 1] = vec.at(Rank - 1);
if (Rank > 1) {
vector_to_coord<TensorCoord, Rank - 1>(coord, vec);
}
}
};
/// Permits copying dynamic vectors into static-length vectors
template <typename TensorCoord>
struct vector_to_coord<TensorCoord, 1> {
vector_to_coord(TensorCoord &coord, std::vector<int> const &vec) {
coord[0] = vec.at(0);
}
};
/// Permits copying dynamic vectors into static-length vectors
template <typename TensorCoord>
struct vector_to_coord<TensorCoord, 0> {
vector_to_coord(TensorCoord &coord, std::vector<int> const &vec) {
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename T>
std::ostream &operator<<(std::ostream &out, std::vector<T> const &vec) {
auto it = vec.begin();
if (it != vec.end()) {
out << *it;
for (++it; it != vec.end(); ++it) {
out << ", " << *it;
}
}
return out;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Permits copying static-length vectors into dynamic vectors
template <typename TensorCoord, int Rank>
struct coord_to_vector {
coord_to_vector(std::vector<int> &vec, TensorCoord const &coord) {
vec.at(Rank - 1) = coord[Rank - 1];
coord_to_vector<TensorCoord, Rank - 1>(vec, coord);
}
};
/// Permits copying static-length vectors into dynamic vectors
template <typename TensorCoord>
struct coord_to_vector<TensorCoord, 1> {
coord_to_vector(std::vector<int> &vec, TensorCoord const &coord) {
vec.at(0) = coord[0];
}
};
/// Permits copying static-length vectors into dynamic vectors
template <typename TensorCoord>
struct coord_to_vector<TensorCoord, 0> {
coord_to_vector(std::vector<int> &vec, TensorCoord const &coord) {
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Structure representing an element in source memory
struct Element {
std::vector<int> coord; ///< logical coordinate of element (as vector)
int offset; ///< linear offset from source memory
int color; ///< enables coloring each element to indicate
/// Default ctor
inline Element(): offset(-1), color(0) { }
/// Construct from logical coordinate and initial offset
inline Element(
std::vector<int> const &coord_,
int offset_,
int color_ = 0
):
coord(coord_), offset(offset_), color(color_) { }
/// Returns true if element is in a defined state
inline bool valid() const {
return offset >= 0;
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Visualizes memory layouts by constructing a 'shape'
template <typename Layout_>
class VisualizeLayout : public VisualizeLayoutBase {
public:
using Layout = Layout_;
using TensorCoord = typename Layout::TensorCoord;
using Stride = typename Layout::Stride;
public:
Options options;
Layout layout;
TensorCoord extent;
std::vector<Element> elements;
public:
/// Initializes the problem space
VisualizeLayout() {
}
/// visualization method
bool visualize(Options const &options_) {
options = options_;
if (options.extent.size() != TensorCoord::kRank) {
std::cerr
<< "--extent must have rank " << TensorCoord::kRank
<< " (given: " << options.extent.size() << ")" << std::endl;
return false;
}
vector_to_coord<TensorCoord, TensorCoord::kRank>(extent, options.extent);
// Construct the layout for a packed tensor
if (options.stride.empty()) {
layout = Layout::packed(extent);
}
else if (options.stride.size() != Stride::kRank) {
std::cerr
<< "--stride must have rank " << Stride::kRank
<< " (given: " << options.stride.size() << ")" << std::endl;
return false;
}
else {
// Stride from
Stride stride;
vector_to_coord<Stride, Stride::kRank>(stride, options.stride);
layout = Layout(stride);
}
// Resize elements, setting elements to 'undefined' state
elements.resize(layout.capacity(extent));
// enumerate points in tensor space and assign
cutlass::reference::host::TensorForEachLambda(
extent,
[&](TensorCoord coord) {
std::vector<int> coord_vec(TensorCoord::kRank, 0);
coord_to_vector<TensorCoord, TensorCoord::kRank>(coord_vec, coord);
int offset = int(layout(coord));
if (offset >= int(elements.size())) {
std::cerr
<< "Layout error - " << coord_vec
<< " is out of range (computed offset: " << offset
<< ", capacity: " << elements.size() << std::endl;
throw std::out_of_range("(TensorForEach) layout error - coordinate out of range");
}
elements.at(offset) = Element(coord_vec, offset);
});
return true;
}
/// Verifies the layout satisfies vectorization requirements
bool verify(bool verbose, std::ostream &out) {
return true;
}
private:
/// returns a pair (is_vectorizable, one_changing_rank) to determine if a
/// vector exists (consecutive logical coordinates or uniformly invalid)
/// at the given location.
std::pair< bool, int > _is_vectorizable(int i) const {
// (all elements are invalid) or
// (all elements are valid AND
// exactly one rank is changing AND
// elements are consecutive)
// Don't need vectorization.
if (options.vectorize <= 2) return std::make_pair(false, -1);
// Boundary check.
if (i > elements.size() || (i + options.vectorize - 1) > elements.size())
return std::make_pair(false, -1);
// Check if either all elements are valid or invalid.
bool all_elements_invalid = std::all_of(
elements.begin() + i, elements.begin() + i + options.vectorize,
[](Element const &e) { return !e.valid(); });
bool all_elements_valid = std::all_of(
elements.begin() + i, elements.begin() + i + options.vectorize,
[](Element const &e) { return e.valid(); });
if (!all_elements_invalid && !all_elements_valid)
return std::make_pair(false, -1);
// From here, it is vectorizable.
if (all_elements_invalid) return std::make_pair(true, -1);
// Check if only exactly one rank is changing.
int one_changing_rank = -1;
for (int j = 0; j < options.vectorize; ++j) {
for (int r = 0; r < TensorCoord::kRank; ++r) {
if (elements.at(i + j).coord.at(r) != elements.at(i).coord.at(r)) {
if (one_changing_rank == -1) {
one_changing_rank = r;
} else if (one_changing_rank != r) {
return std::make_pair(false, -1);
}
}
}
}
return std::make_pair(true, one_changing_rank);
}
/// Prints a vector of elements
void _print_vector(std::ostream &out, int i, int one_changing_rank) {
Element const &base_element = elements.at(i);
if (base_element.valid()) {
out << "(";
for (int r = 0; r < TensorCoord::kRank; ++r) {
if (r) {
out << ", ";
}
if (r == one_changing_rank) {
out
<< base_element.coord.at(r)
<< ".."
<< (base_element.coord.at(r) + options.vectorize - 1);
}
else {
out << base_element.coord.at(r);
}
}
out << ")";
}
else {
out << " ";
}
}
/// Prints a single element
void _print_element(std::ostream &out, int k) {
Element const &element = elements.at(k);
if (element.valid()) {
out << "(";
for (int v = 0; v < TensorCoord::kRank; ++v) {
out << (v ? ", " : "") << element.coord.at(v);
}
out << ")";
}
else {
out << " ";
}
}
public:
/// Pretty-prints the layout to the console
void print_csv(std::ostream &out, char delim = '|', char new_line = '\n') {
int row = -1;
for (int i = 0; i < int(elements.size()); i += options.vectorize) {
if (i % options.output_shape.at(0)) {
out << delim;
}
else {
if (row >= 0) {
out << new_line;
}
++row;
if (row == options.output_shape.at(1)) {
out << new_line;
row = 0;
}
}
auto is_vector = _is_vectorizable(i);
if (is_vector.first) {
_print_vector(out, i, is_vector.second); // print a vector starting at element i
}
else {
for (int j = 0; j < options.vectorize; ++j) { // print individual elements [i..i+j)
_print_element(out, i + j);
}
}
}
out << new_line << std::flush;
}
/// Help message
virtual std::ostream &print_help(std::ostream &out) {
out << "TensorCoord rank " << TensorCoord::kRank << ", Stride rank: " << Stride::kRank;
return out;
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
examples/04_tile_iterator/CMakeLists.txt
+3 -15
View File
@@ -20,19 +20,7 @@
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
set(EXAMPLES_BASIC_CUTLASS_GEMM_SOURCES
tile_iterator.cu
)
if (NOT CUTLASS_NATIVE_CUDA)
# cuda_add_executable does not take interface include directories into account
# Let's fetch them and pass them to CUDA.
get_target_property(CUTLASS_INCLUDES CUTLASS INTERFACE_INCLUDE_DIRECTORIES)
include_directories("${CUTLASS_INCLUDES}")
endif()
cutlass_add_executable(
cutlass_example_add_executable(
04_tile_iterator
${EXAMPLES_BASIC_CUTLASS_GEMM_SOURCES}
)
tile_iterator.cu
)
examples/04_tile_iterator/tile_iterator.cu
+130 -162
View File
@@ -24,225 +24,193 @@
**************************************************************************************************/
/*
This example demonstrates how to use the TileIterator in CUTLASS to load data from addressable
memory, and store it back into addressable memory.
This example demonstrates how to use the PredicatedTileIterator in CUTLASS to load data from
addressable memory, and then store it back into addressable memory.
TileIterator is a core concept in CUTLASS that enables efficient loading and storing of data from
and to addressable memory. The TileIterator accepts a TileTraits type, which defines the shape of a
tile and the distribution of accesses by individual entities, either threads or others.
TileIterator is a core concept in CUTLASS that enables efficient loading and storing of data to
and from addressable memory. The PredicateTileIterator accepts a ThreadMap type, which defines
the mapping of threads to a "tile" in memory. This separation of concerns enables user-defined
thread mappings to be specified.
In this example, a LoadTileIterator is used to load elements from a tile in global memory, stored in
column-major layout, into a fragment, and a corresponding StoreTileIterator is used to store the
elements back into global memory (in the same column-major layout).
https://devblogs.nvidia.com/cutlass-linear-algebra-cuda/
In this example, a PredicatedTileIterator is used to load elements from a tile in global memory,
stored in column-major layout, into a fragment and then back into global memory in the same
layout.
This example uses CUTLASS utilities to ease the matrix operations.
*/
// Standard Library includes
#include <iostream>
#include <sstream>
#include <vector>
#include <fstream>
// CUTLASS includes
#include "cutlass/tile_iterator.h"
#include "cutlass/tile_traits_standard.h"
#include "cutlass/transform/threadblock/predicated_tile_iterator.h"
#include "cutlass/layout/pitch_linear.h"
#include "cutlass/transform/pitch_linear_thread_map.h"
//
// CUTLASS utility includes
// CUTLASS utility includes
//
// Defines operator<<() to write TensorView objects to std::ostream
#include "tools/util/tensor_view_io.h"
#include "cutlass/util/tensor_view_io.h"
// Defines cutlass::HostMatrix<>
#include "tools/util/host_matrix.h"
// Defines cutlass::HostTensor<>
#include "cutlass/util/host_tensor.h"
// Defines cutlass::reference::device::TensorInitialize()
#include "tools/util/reference/device/tensor_elementwise.h"
// Defines cutlass::reference::host::TensorFill() and
// cutlass::reference::host::TensorFillBlockSequential()
#include "cutlass/util/reference/host/tensor_fill.h"
// Defines cutlass::reference::host::TensorEquals()
#include "tools/util/reference/host/tensor_elementwise.h"
///////////////////////////////////////////////////////////////////////////////////////////////////
//
// This function defines load and store tile iterators to load and store a M-by-K tile, in
// column-major layout, from and back into global memory.
//
#pragma warning( disable : 4503)
///////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Traits>
__global__ void cutlass_tile_iterator_load_store_global(
float const *input,
float *output,
int M,
int K) {
/// Define PredicatedTileIterators to load and store a M-by-K tile, in column major layout.
// Define a tile load iterator
typedef cutlass::TileLoadIterator<
Traits, // the Traits type, defines shape/distribution of accesses
float, // elements are of type float
cutlass::IteratorAdvance::kH, // post-increment accesses advance in strided (as opposed to
// contiguous dimension
cutlass::MemorySpace::kGlobal // iterator loads from global memory
> TileLoadIterator;
template <typename Iterator>
__global__ void copy(
typename Iterator::Params dst_params,
typename Iterator::Element *dst_pointer,
typename Iterator::Params src_params,
typename Iterator::Element *src_pointer,
cutlass::Coord<2> extent) {
// Defines a tile store iterator
typedef cutlass::TileStoreIterator<
Traits, // the Traits type, defines shape/distribution of accesses
float, // elements are of type float
cutlass::IteratorAdvance::kH, // post-increment accesses advance in strided (as opposed to
// contiguous) dimension
cutlass::MemorySpace::kGlobal // iterator stores into global memory
> TileStoreIterator;
// Defines a predicate vector for managing statically sized vector of boolean predicates
typedef typename TileLoadIterator::PredicateVector PredicateVector;
Iterator dst_iterator(dst_params, dst_pointer, extent, threadIdx.x);
Iterator src_iterator(src_params, src_pointer, extent, threadIdx.x);
// The parameters specified to the iterators. These include the pointer to the source of
// addressable memory, and the strides and increments for each of the tile's dimensions
typename TileLoadIterator::Params load_params;
typename TileStoreIterator::Params store_params;
// PredicatedTileIterator uses PitchLinear layout and therefore takes in a PitchLinearShape.
// The contiguous dimension can be accessed via Iterator::Shape::kContiguous and the strided
// dimension can be accessed via Iterator::Shape::kStrided
int iterations = (extent[1] + Iterator::Shape::kStrided - 1) / Iterator::Shape::kStrided;
// Initializing the parameters for both of the iterators. The TileLoadIterator accesses the
// input matrix and TileStoreIterator accesses the output matrix. The strides are set
// identically since the data is being stored in the same way as it is loaded (column-major
// mapping).
load_params.initialize(input, M*K, M, 1);
store_params.initialize(output, M*K, M, 1);
// Constructing the tile load and store iterators, and the predicates vector
TileLoadIterator load_iterator(load_params);
TileStoreIterator store_iterator(store_params);
PredicateVector predicates;
typename Iterator::Fragment fragment;
// Initializing the predicates with bounds set to <1, K, M>. This protects out-of-bounds loads.
load_iterator.initialize_predicates(predicates.begin(), cutlass::make_Coord(1, K, M));
for(int i = 0; i < fragment.size(); ++i) {
fragment[i] = 0;
}
// The fragment in which the elements are loaded into and stored from.
typename TileLoadIterator::Fragment fragment;
src_iterator.load(fragment);
dst_iterator.store(fragment);
// Loading a tile into a fragment and advancing to the next tile's position
load_iterator.load_post_increment(fragment, predicates.begin());
// Storing a tile from fragment and advancing to the next tile's position
store_iterator.store_post_increment(fragment);
++src_iterator;
++dst_iterator;
for(; iterations > 1; --iterations) {
src_iterator.load(fragment);
dst_iterator.store(fragment);
++src_iterator;
++dst_iterator;
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// Launches cutlass_tile_iterator_load_store_global kernel
cudaError_t test_cutlass_tile_iterator() {
cudaError_t result = cudaSuccess;
// Initializes the source tile with sequentially increasing values and performs the copy into
// the destination tile using two PredicatedTileIterators, one to load the data from addressable
// memory into a fragment (regiser-backed array of elements owned by each thread) and another to
// store the data from the fragment back into the addressable memory of the destination tile.
// Creating a M-by-K (128-by-8) tile for this example.
static int const M = 128;
static int const K = 8;
// The kernel is launched with 128 threads per thread block.
static int const kThreadsPerThreadBlock = 128;
// Define the tile type
typedef cutlass::Shape<1, 8, 128> Tile;
cudaError_t TestTileIterator(int M, int K) {
// CUTLASS provides a standard TileTraits type, which chooses the 'best' shape to enable warp
// raking along the contiguous dimension if possible.
typedef cutlass::TileTraitsStandard<Tile, kThreadsPerThreadBlock> Traits;
// For this example, we chose a <64, 4> tile shape. The PredicateTileIterator expects
// PitchLinearShape and PitchLinear layout.
using Shape = cutlass::layout::PitchLinearShape<64, 4>;
using Layout = cutlass::layout::PitchLinear;
using Element = int;
int const kThreads = 32;
// M-by-K input matrix of float
cutlass::HostMatrix<float> input(cutlass::MatrixCoord(M, K));
// ThreadMaps define how threads are mapped to a given tile. The PitchLinearStripminedThreadMap
// stripmines a pitch-linear tile among a given number of threads, first along the contiguous
// dimension then along the strided dimension.
using ThreadMap = cutlass::transform::PitchLinearStripminedThreadMap<Shape, kThreads>;
// M-by-K output matrix of float
cutlass::HostMatrix<float> output(cutlass::MatrixCoord(M, K));
// Define the PredicateTileIterator, using TileShape, Element, Layout, and ThreadMap types
using Iterator = cutlass::transform::threadblock::PredicatedTileIterator<
Shape, Element, Layout, 1, ThreadMap>;
//
// Initialize input matrix with linear combination.
//
cutlass::Distribution dist;
cutlass::Coord<2> copy_extent = cutlass::make_Coord(M, K);
cutlass::Coord<2> alloc_extent = cutlass::make_Coord(M, K);
// Linear distribution in column-major format.
dist.set_linear(1, 1, M);
// Allocate source and destination tensors
cutlass::HostTensor<Element, Layout> src_tensor(alloc_extent);
cutlass::HostTensor<Element, Layout> dst_tensor(alloc_extent);
// Arbitrary RNG seed value. Hard-coded for deterministic results.
int seed = 2080;
Element oob_value = Element(-1);
cutlass::reference::device::TensorInitialize(
input.device_view(), // concept: TensorView
seed,
dist);
// Initialize destination tensor with all -1s
cutlass::reference::host::TensorFill(dst_tensor.host_view(), oob_value);
// Initialize source tensor with sequentially increasing values
cutlass::reference::host::BlockFillSequential(src_tensor.host_data(), src_tensor.capacity());
// Initialize output matrix to all zeroes.
output.fill(0);
dst_tensor.sync_device();
src_tensor.sync_device();
// Launch kernel to load and store tiles from/to global memory.
cutlass_tile_iterator_load_store_global<Traits><<<
dim3(1, 1, 1),
dim3(kThreadsPerThreadBlock, 1)
>>>(input.device_data(), output.device_data(), M, K);
typename Iterator::Params dst_params(dst_tensor.layout());
typename Iterator::Params src_params(src_tensor.layout());
result = cudaDeviceSynchronize();
dim3 block(kThreads, 1);
dim3 grid(1, 1);
if (result != cudaSuccess) {
return result;
}
// Launch copy kernel to perform the copy
copy<Iterator><<< grid, block >>>(
dst_params,
dst_tensor.device_data(),
src_params,
src_tensor.device_data(),
copy_extent
);
// Copy results to host
output.sync_host();
cudaError_t result = cudaGetLastError();
if(result != cudaSuccess) {
std::cerr << "Error - kernel failed." << std::endl;
return result;
}
// Verify results
for(int i = 0; i < M; ++i) {
for(int j = 0; j < K; ++j) {
if(output.at(cutlass::make_Coord(i, j)) != float(M*j+i+1)){
std::cout << "FAILED: (" << i << ", " << j
<< ") -- expected: " << (M*j+i+1)
<< ", actual: " << output.at(cutlass::make_Coord(i, j))
<< std::endl;
result = cudaErrorUnknown;
break;
dst_tensor.sync_host();
// Verify results
for(int s = 0; s < alloc_extent[1]; ++s) {
for(int c = 0; c < alloc_extent[0]; ++c) {
Element expected = Element(0);
if(c < copy_extent[0] && s < copy_extent[1]) {
expected = src_tensor.at({c, s});
}
else {
expected = oob_value;
}
Element got = dst_tensor.at({c, s});
bool equal = (expected == got);
if(!equal) {
std::cerr << "Error - source tile differs from destination tile." << std::endl;
return cudaErrorUnknown;
}
}
}
}
return result;
return cudaSuccess;
}
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Entry point to tile_iterator example.
//
// usage:
//
// 04_tile_iterator
//
int main(int argc, const char *arg[]) {
// Properties of CUDA device
cudaDeviceProp device_properties;
// Assumne the device id is 0.
int device_id = 0;
cudaError_t result = cudaGetDeviceProperties(&device_properties, device_id);
if (result != cudaSuccess) {
std::cerr << "Failed to get device properties: "
<< cudaGetErrorString(result) << std::endl;
return -1;
}
cudaError_t result = TestTileIterator(57, 35);
if(result == cudaSuccess) {
std::cout << "Passed." << std::endl;
}
//
// Run the CUTLASS tile iterator test.
//
result = test_cutlass_tile_iterator();
if (result == cudaSuccess) {
std::cout << "Passed." << std::endl;
}
// Exit.
return result == cudaSuccess ? 0 : -1;
// Exit
return result == cudaSuccess ? 0 : -1;
}
///////////////////////////////////////////////////////////////////////////////////////////////////
examples/03_strided_batched_gemm/CMakeLists.txt → examples/05_batched_gemm/CMakeLists.txt
+4 -15
View File
@@ -20,19 +20,8 @@
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
set(EXAMPLES_STRIDED_BATCHED_GEMM_SOURCES
strided_batched_gemm.cu
)
if (NOT CUTLASS_NATIVE_CUDA)
# cuda_add_executable does not take interface include directories into account
# Let's fetch them and pass them to CUDA.
get_target_property(CUTLASS_INCLUDES CUTLASS INTERFACE_INCLUDE_DIRECTORIES)
include_directories("${CUTLASS_INCLUDES}")
endif()
cutlass_add_executable(
03_strided_batched_gemm
${EXAMPLES_STRIDED_BATCHED_GEMM_SOURCES}
cutlass_example_add_executable(
05_batched_gemm
batched_gemm.cu
)
examples/03_strided_batched_gemm/strided_batched_gemm.cu → examples/05_batched_gemm/batched_gemm.cu
+58 -64
View File
@@ -25,9 +25,10 @@
#include <iostream>
#include <vector>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/sgemm_traits.h"
#include "cutlass/layout/matrix.h"
#include "cutlass/gemm/device/gemm_batched.h"
#pragma warning( disable : 4503)
@@ -88,7 +89,12 @@ The stride (batch_stride_C) between the first element of two batches is k
*/
cudaError_t cutlass_strided_batched_sgemm(float const *A,
cudaError_t cutlass_strided_batched_sgemm(
int m,
int n,
int k,
float alpha,
float const *A,
int lda,
long long int batch_stride_A,
float const *B,
@@ -97,60 +103,45 @@ cudaError_t cutlass_strided_batched_sgemm(float const *A,
float *C,
int ldc,
long long int batch_stride_C,
float alpha,
float beta,
int m,
int n,
int k,
int batch_count) {
// create a cutlass traits
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
// create a CUTLASS GEMM object.
typedef cutlass::gemm::Gemm<SgemmTraits> Gemm;
// Construct and initialize CUTLASS GEMM parameters object.
typename Gemm::Params params;
int result = params.initialize(
m, // M dimension for each batch
n, // N dimension for each batch
k, // K dimension for each batch
alpha, // scalar alpha
A,
lda,
batch_stride_A, // distance in memory between the first element of neighboring batch
B,
ldb,
batch_stride_B, // distance in memory between the first element of neighboring batch
beta, // scalar beta
C, // source matrix C
ldc,
batch_stride_C, // distance in memory between the first element of neighboring batch
C, // destination matrix C (may be different memory than source C matrix)
ldc,
batch_stride_C, // distance in memory between the first element of neighboring batch
using Gemm = cutlass::gemm::device::GemmBatched<
float, cutlass::layout::ColumnMajor,
float, cutlass::layout::ColumnMajor,
float, cutlass::layout::ColumnMajor
>;
Gemm gemm_op;
cutlass::Status status = gemm_op({
{m, n, k},
{A, lda},
batch_stride_A,
{B, ldb},
batch_stride_B,
{C, ldc},
batch_stride_C,
{C, ldc},
batch_stride_C,
{alpha, beta},
batch_count
);
if (result != 0) {
std::cerr << "Failed to initialize CUTLASS Gemm::Params object." << std::endl;
return cudaErrorInvalidValue;
});
if (status != cutlass::Status::kSuccess) {
return cudaErrorUnknown;
}
// Launch the CUTLASS GEMM kernel.
Gemm::launch(params);
result = cudaDeviceSynchronize();
if (result != cudaSuccess) {
std::cerr << "kernel launch result = " << result << std::endl;
}
return cudaGetLastError();
return cudaSuccess;
}
template<typename T>
cudaError_t strided_batched_gemm_nn_reference(std::vector<T> const &A,
cudaError_t strided_batched_gemm_nn_reference(
int m,
int n,
int k,
T alpha,
std::vector<T> const &A,
int lda,
long long int batch_stride_A,
std::vector<T> const &B,
@@ -159,11 +150,7 @@ cudaError_t strided_batched_gemm_nn_reference(std::vector<T> const &A,
std::vector<T> &C,
int ldc,
long long int batch_stride_C,
T alpha,
T beta,
int m,
int n,
int k,
int batch_count) {
/*
strided batched gemm NN
@@ -202,10 +189,12 @@ cudaError_t strided_batched_gemm_nn_reference(std::vector<T> const &A,
}
int main() {
int const m = 16;
int const n = 24;
int const k = 8;
int const batch_count = 3;
// Arbitrary problem size
int const m = 520;
int const n = 219;
int const k = 129;
int const batch_count = 17;
// A, B are non-transpose, column major
int const lda = m;
@@ -254,11 +243,14 @@ int main() {
return result;
}
// Limit range to avoid floating-point errors
int const kRange = 8;
// fill A
for (int b_idx = 0; b_idx < batch_count; b_idx++) {
for (int col_idx = 0; col_idx < k; col_idx++) {
for (int row_idx = 0; row_idx < m; row_idx++) {
host_A[row_idx + col_idx * lda + b_idx * lda * k] = static_cast<float>(row_idx + col_idx * lda + b_idx * lda * k);
host_A[row_idx + col_idx * lda + b_idx * lda * k] = static_cast<float>((row_idx + col_idx * lda + b_idx * lda * k) % kRange);
}
}
}
@@ -266,7 +258,7 @@ int main() {
for (int b_idx = 0; b_idx < batch_count; b_idx++) {
for (int col_idx = 0; col_idx < n; col_idx++) {
for (int row_idx = 0; row_idx < k; row_idx++) {
host_B[row_idx + col_idx * ldb + b_idx * k] = static_cast<float>(n + k * ldb + batch_count * k) - static_cast<float>(row_idx + col_idx * ldb + b_idx * k);
host_B[row_idx + col_idx * ldb + b_idx * k] = static_cast<float>(((n + k * ldb + batch_count * k) - (row_idx + col_idx * ldb + b_idx * k)) % kRange);
}
}
}
@@ -301,8 +293,9 @@ int main() {
}
// run cutlass
result = cutlass_strided_batched_sgemm(A, lda, batch_stride_A, B, ldb, batch_stride_B, C, ldc, batch_stride_C,
alpha, beta, m, n, k, batch_count);
result = cutlass_strided_batched_sgemm(
m, n, k, alpha, A, lda, batch_stride_A, B, ldb, batch_stride_B, C, ldc, batch_stride_C,
beta, batch_count);
if (result != cudaSuccess)
return result;
@@ -314,11 +307,12 @@ int main() {
}
//compare with reference code
result = strided_batched_gemm_nn_reference(ref_A, lda, batch_stride_A, ref_B, ldb, batch_stride_B, ref_C, ldc, batch_stride_C,
alpha, beta, m, n, k, batch_count);
result = strided_batched_gemm_nn_reference(m, n, k, alpha, ref_A, lda, batch_stride_A, ref_B, ldb, batch_stride_B, ref_C, ldc, batch_stride_C,
beta, batch_count);
if (result != 0)
return result;
// Expect bit-level accuracy for this simple example
if (ref_C != result_C) {
std::cout << "CUTLASS strided batched gemm does not run correctly" << std::endl;
return cudaErrorUnknown;
examples/05_wmma_gemm/wmma_gemm.cu
-355
View File
@@ -1,355 +0,0 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*
This example demonstrates how to call a CUTLASS GEMM kernel using Turing integer WMMA.
The CUTLASS integer WMMA Gemm template is instantiated in the function Cutlass_S8_WmmagemmNN. This
is kernel computes the general matrix product (GEMM) using integer arithmetic accelerated by Turing
WMMA and assumes all matrices have column-major layout.
The threadblock tile size is chosen as 128x128x8 which offers good performance for large matrices.
See the CUTLASS Parallel for All blog post for more exposition on the tunable parameters available
in CUTLASS.
https://devblogs.nvidia.com/cutlass-linear-algebra-cuda/
This example uses CUTLASS utilities to ease the matrix operations.
*/
// Standard Library includes
#include <iostream>
#include <sstream>
#include <vector>
// CUTLASS includes needed for WMMA GEMM kernel
#include "cutlass/wmma_matrix.h"
#pragma warning( disable : 4503)
// This example works only when this MACRO is defined in "cutlass/wmma_matrix.h"
#ifdef CUTLASS_USE_INT_WMMA
// Defines cutlass::gemm::Gemm, the generic Gemm computation template class.
#include "cutlass/gemm/gemm.h"
// Defines cutlass::gemm::WmmaGemmTraits, the structural components for WMMA GEMM
#include "cutlass/gemm/wmma_gemm_traits.h"
//
// CUTLASS utility includes
//
// Defines operator<<() to write TensorView objects to std::ostream
#include "tools/util/tensor_view_io.h"
// Defines cutlass::HostMatrix<>
#include "tools/util/host_matrix.h"
// Defines cutlass::reference::device::TensorInitialize()
#include "tools/util/reference/device/tensor_elementwise.h"
// Defines cutlass::reference::host::TensorEquals()
#include "tools/util/reference/host/tensor_elementwise.h"
// Defines cutlass::reference::host::Gemm()
#include "tools/util/reference/host/gemm.h"
///////////////////////////////////////////////////////////////////////////////////////////////////
//
// This function defines a CUTLASS GEMM kernel instantiation, constructs its parameters object,
// and launches it on the CUDA device.
//
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Define a CUTLASS GEMM template and launch a GEMM kernel.
cudaError_t Cutlass_S8_WmmagemmNN(
int M,
int N,
int K,
int alpha,
signed char const *A,
int lda,
signed char const *B,
int ldb,
int beta,
int *C,
int ldc) {
// Define type definition for 8-bit signed int WMMA CUTLASS GEMM with column-major
// input matrices and 128x128x128 threadblock tile size.
//
// Note, A and B are 8-bit signed int. C and D are 32-bit int. .
//
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor, // layout of A matrix
cutlass::MatrixLayout::kColumnMajor, // layout of B matrix
cutlass::Shape<128, 128, 128>, // threadblock tile size
signed char, // A type
signed char, // B type
int, // D type
cutlass::gemm::LinearScaling<int>, // functor to do the math in the epilogue
int, // accumulator type
cutlass::Shape<128, 32, 32>, // warp tile size
cutlass::Shape<16, 16, 16>, // WMMA instruction tile size
16, // scalars every time a thread loads from A
16 // scalars every time a thread loads from B
>
GemmTraits;
// Define a CUTLASS GEMM type from a GemmTraits<> instantiation.
typedef cutlass::gemm::Gemm<GemmTraits> Gemm;
// Construct and initialize CUTLASS GEMM parameters object.
typename Gemm::Params params;
int result = params.initialize(
M, // GEMM M dimension
N, // GEMM N dimension
K, // GEMM K dimension
alpha, // scalar alpha
A, // matrix A operand
lda,
B, // matrix B operand
ldb,
beta, // scalar beta
C, // source matrix C
ldc,
C, // destination matrix C (may be different memory than source C matrix)
ldc
);
if (result) {
std::cerr << "Failed to initialize CUTLASS Gemm::Params object." << std::endl;
return cudaErrorInvalidValue;
}
// Launch the CUTLASS GEMM kernel.
Gemm::launch(params);
// Return any errors associated with the launch or cudaSuccess if no error.
return cudaGetLastError();
}
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Allocate several matrices in GPU device memory and call an integer
/// CUTLASS WMMA GEMM kernel.
cudaError_t TestCutlassGemm(int M, int N, int K, int alpha, int beta) {
cudaError_t result;
//
// Construct cutlass::HostMatrix<> using the integer host-side types.
// M-by-K matrix of signed char
cutlass::HostMatrix<signed char> A(cutlass::MatrixCoord(M, K));
// K-by-N matrix of signed char
cutlass::HostMatrix<signed char> B(cutlass::MatrixCoord(K, N));
// M-by-N matrix of int
cutlass::HostMatrix<int> C_cutlass(cutlass::MatrixCoord(M, N));
// M-by-N matrix of int
cutlass::HostMatrix<int> C_reference(cutlass::MatrixCoord(M, N));
//
// Initialize matrices with small, random integers.
//
cutlass::Distribution dist;
// Uniform random distribution from -4 .. 4. Values are truncated to integers.
dist.set_uniform(-4, 4);
// Arbitrary RNG seed value. Hard-coded for deterministic results.
int seed = 2080;
cutlass::reference::device::TensorInitialize(
A.device_view(), // concept: TensorView
seed,
dist);
cutlass::reference::device::TensorInitialize(
B.device_view(), // concept: TensorView
seed * 2,
dist);
cutlass::reference::device::TensorInitialize(
C_cutlass.device_view(), // concept: TensorView
seed * 3,
dist);
// Copy C_cutlass into C_reference so the GEMM is correct when beta != 0.
cutlass::reference::device::TensorFill(C_reference.device_view(), C_cutlass.device_view());
// Copy the device-side view into host memory
C_reference.sync_host();
//
// Launch the CUTLASS GEMM kernel
//
result = Cutlass_S8_WmmagemmNN(
M,
N,
K,
alpha,
A.device_data(),
A.leading_dim(),
B.device_data(),
B.leading_dim(),
beta,
C_cutlass.device_data(),
C_cutlass.leading_dim()
);
if (result != cudaSuccess) {
return result;
}
//
// Verify the result using a host-side reference
//
// A and B were initialized using device-side procedures.
A.sync_host();
B.sync_host();
// Copy CUTLASS's GEMM results into host memory.
C_cutlass.sync_host();
// Compute the reference result using the host-side GEMM reference implementation.
cutlass::reference::host::Gemm(
cutlass::gemm::GemmCoord(K, N, M), // problem size (type: cutlass::gemm::GemmCoord)
alpha, // alpha (type: int)
A.host_ref(), // A (concept: TensorRef)
B.host_ref(), // B (concept: TensorRef)
beta, // beta (int)
C_reference.host_ref(), // C (concept: TensorRef)
int(0) // Accumulator initial value passed as argument to deduce
); // internal accumulation data type as int.
// Compare reference to computed results.
if (!cutlass::reference::host::TensorEquals(C_reference.host_view(), C_cutlass.host_view())) {
std::cerr << "Error - CUTLASS WMMA GEMM kernel differs from reference." << std::endl;
//
// On error, print C_cutlass and C_reference to std::cerr.
//
// Result of CUTLASS WMMA GEMM kernel
std::cerr << "CUTLASS:\n" << C_cutlass << std::endl;
// Result of reference computation
std::cerr << "Reference:\n" << C_reference << std::endl;
// Return error code.
return cudaErrorUnknown;
}
// Passed error check
return cudaSuccess;
}
#endif // defined CUTLASS_USE_INT_WMMA
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Entry point to wmma_gemm example.
//
// usage:
//
// 05_wmma_gemm <M> <N> <K> <alpha> <beta>
//
int main(int argc, const char *arg[]) {
#ifdef CUTLASS_USE_INT_WMMA
// Properties of CUDA device
cudaDeviceProp device_properties;
// Assumne the device id is 0.
int device_id = 0;
cudaError_t result = cudaGetDeviceProperties(&device_properties, device_id);
if (result != cudaSuccess) {
std::cerr << "Failed to get device properties: "
<< cudaGetErrorString(result) << std::endl;
return -1;
}
if ((device_properties.major * 10 + device_properties.minor) < 72) {
std::cerr << "This example needs to run on a device which has at least 7.2 compute capability." << std::endl;
return -1;
}
//
// Parse the command line to obtain GEMM dimensions and scalar values.
//
// GEMM problem dimensions.
int problem[3] = { 128, 128, 128 };
for (int i = 1; i < argc && i < 4; ++i) {
std::stringstream ss(arg[i]);
ss >> problem[i - 1];
}
// Scalars used for linear scaling the result of the matrix product.
int scalars[2] = { 1, 0 };
for (int i = 4; i < argc && i < 6; ++i) {
std::stringstream ss(arg[i]);
ss >> scalars[i - 4];
}
//
// Run the CUTLASS GEMM test.
//
result = TestCutlassGemm(
problem[0], // GEMM M dimension
problem[1], // GEMM N dimension
problem[2], // GEMM K dimension
scalars[0], // alpha
scalars[1] // beta
);
if (result == cudaSuccess) {
std::cout << "Passed." << std::endl;
}
// Exit.
return result == cudaSuccess ? 0 : -1;
#else
std::cerr << "CUTLASS WMMA GEMM targeting Turing Tensor Cores features requires compute capability 7.2." << std::endl;
return -1;
#endif // defined CUTLASS_USE_INT_WMMA
}
///////////////////////////////////////////////////////////////////////////////////////////////////
examples/06_splitK_gemm/CMakeLists.txt
+4 -15
View File
@@ -20,19 +20,8 @@
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
set(EXAMPLES_SPLITK_GEMM_SOURCES
splitK_gemm.cu
)
if (NOT CUTLASS_NATIVE_CUDA)
# cuda_add_executable does not take interface include directories into account
# Let's fetch them and pass them to CUDA.
get_target_property(CUTLASS_INCLUDES CUTLASS INTERFACE_INCLUDE_DIRECTORIES)
include_directories("${CUTLASS_INCLUDES}")
endif()
cutlass_add_executable(
cutlass_example_add_executable(
06_splitK_gemm
${EXAMPLES_SPLITK_GEMM_SOURCES}
)
splitk_gemm.cu
)
examples/06_splitK_gemm/splitK_gemm.cu
-298
View File
@@ -1,298 +0,0 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include <iostream>
#include <vector>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device_gemm.h"
#include "cutlass/gemm/sgemm_traits.h"
#include "cutlass/reduction/batched_reduction_traits.h"
#include "cutlass/gemm/device_gemm_traits.h"
#pragma warning( disable : 4503)
/*
This example demonstrates how to use cutlass to compute sgemm with splitK
splitK is useful for gemm with small M and N and reasonably large K.
Because the sizes of M and N are small, the number of threadblocks we can launch is often limited and
results in under utilization of the hardware.
splitK allows us to divide a gemm across K dimension by first launching a partitionedK gemm (very similar to batched gemm),
storing the intermediate result in workspace and then launching a second reduction kernel.
Thus, as demonstrated by function cutlass_splitK_sgemm_nn(), the users need to create two traits, one for the partitionedK gemm,
and one for the reduction. The users are also responsible for allocating and releasing the workspace memory. The size of the workspace
memory can be queried by calling required_workspace_memory_in_byte().
*/
template<int splits_count>
cudaError_t cutlass_splitK_sgemm_nn(float const *A,
int lda,
float const *B,
int ldb,
float *C,
int ldc,
float alpha,
float beta,
int m,
int n,
int k) {
cudaError_t result = cudaSuccess;
// create cutlass gemm traits for the first kernel
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor, /*the layout of A*/
cutlass::MatrixLayout::kColumnMajor, /*the layout of B*/
cutlass::Shape<8, 128, 128> > /*the tile for each threadblock*/
SgemmTraits;
// create cutlass batched reduction traits for the second kernel
// for reduction D = alpha * Reduction(A) + beta * C
typedef cutlass::reduction::BatchedReductionTraits<float, /*the scalar type of A in reduction, not to be confused with A in GEMM*/
float, /*the scalar type of C in reduction, not to be confused with C in GEMM*/
float, /*the scalar type of D in reduction, not to be confused with D in GEMM*/
float, /*the scalar type of alpha and beta in reduction*/
float, /*the scalar type of accumulation in reduction*/
splits_count /*reduction workload*/
>
BatchedReductionTraits;
// create a device gemm that packages gemm traits and batched reduction traits
typedef cutlass::gemm::SplitkPIGemmTraits<SgemmTraits, BatchedReductionTraits> deviceGemmTraits;
// kernel class
typedef typename deviceGemmTraits::KernelClass deviceGemm;
// Params ctor requires M, N, K sizes
typename deviceGemm::Params deviceGemmParams(m, n, k);
// query if workspace is needed. the workspace size is sizeof(accumulateType) * M * N * splits_count
size_t workspace_size = deviceGemmParams.required_workspace_memory_in_byte();
// allocate workspace memory
float *workspace_ptr;
result = cudaMalloc(&workspace_ptr, workspace_size);
if (result != cudaSuccess) {
std::cerr << "cudaMalloc result = " << result << std::endl;
return result;
}
// finish init Params
deviceGemmParams.initialize(alpha, /*alpha*/
A, /*A*/
lda, /*lda*/
B, /*B*/
ldb, /*ldb*/
beta, /*beta*/
C, /*C*/
ldc, /*ldc*/
C, /*D, can point to the same memory with C*/
ldc, /*ldc*/
workspace_ptr /*ptr to workspace*/
);
// launch the kernel
deviceGemm::launch(deviceGemmParams);
result = cudaDeviceSynchronize();
if (result != cudaSuccess) {
std::cerr << "launch result = " << result << std::endl;
cudaFree(workspace_ptr);
return result;
}
// release the workspace memory
result = cudaFree(workspace_ptr);
if (result != cudaSuccess) {
std::cerr << "cudaFree result = " << result << std::endl;
}
return cudaGetLastError();
}
template<typename T>
cudaError_t sgemm_nn_reference(std::vector<T> const &A,
int lda,
std::vector<T> const &B,
int ldb,
std::vector<T> &C,
int ldc,
T alpha,
T beta,
int m,
int n,
int k) {
/*
sgemm
*/
cudaError_t result = cudaSuccess;
for (int n_idx = 0; n_idx < n; n_idx++) {
for (int m_idx = 0; m_idx < m; m_idx++) {
T accum = beta * C[n_idx * ldc + m_idx];
for (int k_idx = 0; k_idx < k; k_idx++) {
accum += alpha
* A[k_idx * lda + m_idx]
* B[n_idx * ldb + k_idx];
}
C[n_idx * ldc + m_idx] = accum;
}
}
return result;
}
int main() {
int const m = 128;
int const n = 128;
int const k = 4096;
//splits_count should be known at compile time
int const splits_count = 80;
// A, B are non-transpose, column major
int const lda = m;
int const ldb = k;
int const ldc = m;
int const count_A = lda * k;
int const count_B = ldb * n;
int const count_C = ldc * n;
// alpha and beta
float alpha = 1.0f;
float beta = 2.0f;
cudaError_t result = cudaSuccess;
// allocate the host memory
std::vector<float> host_A(count_A);
std::vector<float> host_B(count_B);
std::vector<float> host_C(count_C);
std::vector<float> result_C(count_C);
// allocate the device memory
float *A;
float *B;
float *C;
result = cudaMalloc(&A, count_A * sizeof(float));
if (result != cudaSuccess) {
std::cerr << "cudaMalloc result = " << result << std::endl;
return result;
}
result = cudaMalloc(&B, count_B * sizeof(float));
if (result != cudaSuccess) {
std::cerr << "cudaMalloc result = " << result << std::endl;
return result;
}
result = cudaMalloc(&C, count_C * sizeof(float));
if (result != cudaSuccess) {
std::cerr << "cudaMalloc result = " << result << std::endl;
return result;
}
// fill A
for (int col_idx = 0; col_idx < k; col_idx++) {
for (int row_idx = 0; row_idx < m; row_idx++) {
host_A[row_idx + col_idx * lda] = static_cast<float>((row_idx + col_idx) % 10);
}
}
// fill B
for (int col_idx = 0; col_idx < n; col_idx++) {
for (int row_idx = 0; row_idx < k; row_idx++) {
host_B[row_idx + col_idx * ldb] = static_cast<float>((row_idx - col_idx) % 5);
}
}
// fill C
for (int col_idx = 0; col_idx < n; col_idx++) {
for (int row_idx = 0; row_idx < m; row_idx++) {
host_C[row_idx + col_idx * ldc] = 1.f;
}
}
// ref memory
std::vector<float> ref_A(host_A);
std::vector<float> ref_B(host_B);
std::vector<float> ref_C(host_C);
// copy host memory to device
result = cudaMemcpy(A, host_A.data(), count_A * sizeof(float), cudaMemcpyHostToDevice);
if (result != cudaSuccess) {
std::cerr << "cudaMemcpy result = " << result << std::endl;
return result;
}
result = cudaMemcpy(B, host_B.data(), count_B * sizeof(float), cudaMemcpyHostToDevice);
if (result != cudaSuccess) {
std::cerr << "cudaMemcpy result = " << result << std::endl;
return result;
}
result = cudaMemcpy(C, host_C.data(), count_C * sizeof(float), cudaMemcpyHostToDevice);
if (result != cudaSuccess) {
std::cerr << "cudaMemcpy result = " << result << std::endl;
return result;
}
// run cutlass
result = cutlass_splitK_sgemm_nn<splits_count>(A, lda, B, ldb, C, ldc, alpha, beta, m, n, k);
if (result != cudaSuccess)
return result;
// copy device memory to host
result = cudaMemcpy(result_C.data(), C, count_C * sizeof(float), cudaMemcpyDeviceToHost);
if (result != cudaSuccess) {
std::cerr << "cudaMemcpy result = " << result << std::endl;
return result;
}
//compare with reference code
result = sgemm_nn_reference(ref_A, lda, ref_B, ldb, ref_C, ldc, alpha, beta, m, n, k);
if (result != 0)
return result;
if (ref_C != result_C) {
std::cout << "CUTLASS splitK gemm does not run correctly" << std::endl;
return cudaErrorUnknown;
}
// free memory
result = cudaFree(A);
if (result != cudaSuccess) {
std::cerr << "cudaFree result = " << result << std::endl;
return result;
}
result = cudaFree(B);
if (result != cudaSuccess) {
std::cerr << "cudaFree result = " << result << std::endl;
return result;
}
result = cudaFree(C);
if (result != cudaSuccess) {
std::cerr << "cudaFree result = " << result << std::endl;
return result;
}
if (result == cudaSuccess) {
std::cout << "Passed." << std::endl;
}
// Exit.
return result == cudaSuccess ? 0 : -1;
}
examples/06_splitK_gemm/splitk_gemm.cu
+305
View File
@@ -0,0 +1,305 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/**
This example shows how to use split-k version of matrix multiplication using functions and data
structures provided by CUTLASS; which we run on a NVIDIA Volta GPU.
What is split-k?
Consider a problem size of M = 128, N = 128, K = 4096. In this case, if my thread-block tile size (a
tile can be viewed as a 2d matrix) is 128x128x4096, then we launch a singled a thread-block taking
up a single SM of 84 SMs present on V100. Hence the efficiency of computation is really low. So, how
to solve it? This is where split-k comes in. It is a way of partitioning K-dimension of matrix
multiplication and distribute across multiple SMs and get better efficiency than single SM. In the
above example, we can partition K-dimension with split-k factor of 16 i.e., thread-block tile size
will be 128x128x256 and will be launching on 16 SMs. Once each thread-block computes their partial
inner product (1/16th of output), they accumulate to single output matrix.
Writing a single high performance matrix multiplication kernel is hard but do-able. Whereas writing
high performance kernels at scale which works for multiple problem sizes with good abstractions is
really hard. CUTLASS solves this problem by providing simplified abstractions (knobs) to compose
multiple sections of gemm kernel. When used properly, the kernels can hit peak performance of GPU
easily.
CUTLASS divides a kernel into hierarchical composable sections. Which means, at each thread, warp
and thread-block level, they compute on their own tile-size with higher level of tile sizes being
composed from lower level ones. Multiple thread-tiles (tile size each thread computes) can be used
to form warp-tiles (tile size each warp computes) and multiple warp tiles can be used to compute
threadblock-tile (tile size computed by a threadblock).
In thie example, we split variable initialization into
1. Setting up data properties : describes how matrices are laid out in the memory and how the kernel
can view them (logical to physical mapping)
2. Setting up computation properties : describes how the above set matrices will be used to compute
output of matrix multiplication.
First, we setup the data types of matrices A, B, C and D along with alpha, beta as the equation for
GEMM is D = alpha * A * B + beta * C. In CUTLASS, the kernels first compute A * B and leaves the
rest of the computation to end of the kernel as alpha * X + beta * C is a simple element-wise
operation on X (A * B) and C. We call this as epilogue of kernel. Hence, we setup data types for
alpha and beta to be equal to ElementComputeEpilogue = float. As we want to MMA instructions on
Volta and they support only half-precision floating point (fp16 or half), we use data type for
elements in input matrix A and B as cutlass::half_t. Volta also supports accumulation of partial dot
product to fp32, which can store wider range of numbers, we use it as data type of output matrix
elements and accumulation. We convey this to CUTLASS kernel by initializing template variables
ElementAccumulator (float), ElementComputeEpilogue (float), ElementInputA (cutlass::half_t),
ElementInputB (cutlass::half_t), ElementOutput (float). Communicating just the data type is not
enough. As the data is laid out linearly in memory, we have to convey the layout of matrices. We do
that by initializing template variable LayoutInputA to column major cutlass variable, LayoutInputB
to row major and LayoutOutput to row major. Next, we setup rules to comptue alpha * X + beta * C
which is called epilogue of the kernel. We initialize template variable EpilogueOp, which takes the
data type of output ElementOutput (int32_t), the number of elements per vector memory access (16),
data type of accumulator (int32_t) and data type of computation of linear combination (alpha * X +
beta * C).
Now that we setup the properties of data, we have to setup properties of computation.
Second, we create template variables of tile sizes for thread-block, warp and mma-op to 128x128x32,
64x64x4, 8x8x4 (MxNxK) respectively. When passed to instantiate CUTLASS GEMM kernel, it internally
deduce the amount of threads needed per thread-block, amount of shared memory, storing data in
bank-conflict free manner, and ton of other variables required to compose, intialize and launch a
high performance GEMM kernel. This is the beauty of CUTLASS, it relieves developer from
understanding and coding complicated hardware optimizations which can easily go wrong.
There are few more template variables initialized such as, which threadblock tile of output matrix
is done which threadblock launched on an SM, CUDA SM architecture of GPU you want to run on.
These are all put together to create a template variable which describes CUTLASS GEMM kernel using
cutlass::gemm::device::GemmSplitKParallel template.
The next step is to intialize physical data, instantiate and initialize CUTLASS kernel and run it.
We use CUTLASS utilities to initialize, fill, compare matrices as they are simple and doesn't come
in the way of learning CUTLASS.
Once all the matrices are initialized and filled with data, create arguments tuple to launch CUTLASS
kernel which takes problem size (M = 5120, N = 4096 and K = 4096), matrices, alpha, beta and the
important one, split k-dimension factor. Along with that, we query CUTLASS if any scratch-space
memory required by the kernel we instantiated. If yes, we create it and pass it along with other
arguments created to intialize CUTLASS kernel then, the kernel is launched.
In this example, we later on launch a reference gemm kernel (from CUTLASS utilities) to compare if
the output from CUTLASS kernel is same as reference GEMM kernel.
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_splitk_parallel.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/device/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/tensor_view_io.h"
#include "helper.h"
// The code section below describes datatype for input, output matrices and computation between
// elements in input matrices.
using ElementAccumulator = float; // <- data type of accumulator
using ElementComputeEpilogue = ElementAccumulator; // <- data type of epilogue operations
using ElementInputA = cutlass::half_t; // <- data type of elements in input matrix A
using ElementInputB = cutlass::half_t; // <- data type of elements in input matrix B
using ElementOutput = float; // <- data type of elements in output matrix D
// The code section below describes matrix layout of input and output matrices. Column Major for
// Matrix A, Row Major for Matrix B and Row Major for Matrix C
using LayoutInputA = cutlass::layout::ColumnMajor;
using LayoutInputB = cutlass::layout::RowMajor;
using LayoutOutput = cutlass::layout::RowMajor;
// This code section describes whether you want to use tensor cores or regular SIMT cores on GPU SM
using MMAOp = cutlass::arch::OpClassTensorOp;
// This code section describes CUDA SM architecture number
using SmArch = cutlass::arch::Sm70;
// This code section describes the tile size a thread block will compute
using ShapeMMAThreadBlock =
cutlass::gemm::GemmShape<128, 128, 32>; // <- threadblock tile M = 128, N = 128, K = 32
// This code section describes tile size a warp will compute
using ShapeMMAWarp = cutlass::gemm::GemmShape<64, 64, 32>; // <- warp tile M = 64, N = 64, K = 32
// This code section describes the size of MMA op
using ShapeMMAOp = cutlass::gemm::GemmShape<8, 8, 4>; // <- MMA Op tile M = 8, N = 8, K = 4
// This code section describes how threadblocks are scheduled on GPU
using SwizzleThreadBlock = cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle; // <- ??
// This code section describes ?
using EpilogueOp = cutlass::epilogue::thread::LinearCombination<
ElementOutput, // <- data type of output matrix
128 / cutlass::sizeof_bits<ElementOutput>::value, // <- This is the number of elements per
// vectorized memory access. For half
// precision, it's 8 elements. This becomes
// the vector width of math instructions in
// epilogue too
ElementAccumulator, // <- data type of accumulator
ElementComputeEpilogue>; // <- data type for alpha/beta in linear combination function
// Put all the created template variables to create GemmSplitKParallel template variable
using Gemm = cutlass::gemm::device::GemmSplitKParallel<ElementInputA,
LayoutInputA,
ElementInputB,
LayoutInputB,
ElementOutput,
LayoutOutput,
ElementAccumulator,
MMAOp,
SmArch,
ShapeMMAThreadBlock,
ShapeMMAWarp,
ShapeMMAOp,
EpilogueOp>;
int main() {
cudaDeviceProp props;
CUDA_CHECK(cudaGetDeviceProperties(&props, 0));
if (!(props.major >= 7)) {
std::cerr << "Volta Tensor Ops must be run on a machine with compute capability at least 70."
<< std::endl;
return 0;
}
const int length_m = 5120;
const int length_n = 4096;
const int length_k = 4096;
// Create a tuple of problem size for matrix multiplication
cutlass::gemm::GemmCoord problem_size(length_m, length_n, length_k);
// Initialize tensors using CUTLASS helper functions
cutlass::HostTensor<ElementInputA, LayoutInputA> tensor_a(
problem_size.mk()); // <- Create matrix A with dimensions M x K
cutlass::HostTensor<ElementInputB, LayoutInputB> tensor_b(
problem_size.nk()); // <- Create matrix B with dimensions N x K
cutlass::HostTensor<ElementOutput, LayoutOutput> tensor_c(
problem_size.mn()); // <- Create matrix C with dimensions M x N
cutlass::HostTensor<ElementOutput, LayoutOutput> tensor_d(
problem_size.mn()); // <- Create matrix D with dimensions M x N used to store output from
// CUTLASS kernel
cutlass::HostTensor<ElementOutput, LayoutOutput> tensor_ref_d(
problem_size.mn()); // <- Create matrix D with dimensions M x N used to store output from
// reference kernel
// Fill input and output matrices on host using CUTLASS helper functions
cutlass::reference::host::TensorFillRandomUniform(
tensor_a.host_view(),
1,
ElementInputA(4),
ElementInputA(-4),
0); // <- Fill matrix A on host with uniform-distribution random data
cutlass::reference::host::TensorFillRandomUniform(
tensor_b.host_view(),
1,
ElementInputB(4),
ElementInputB(-4),
0); // <- Fill matrix B on host with uniform-distribution random data
cutlass::reference::host::TensorFillRandomUniform(
tensor_c.host_view(),
1,
ElementOutput(4),
ElementOutput(-4),
0); // <- Fill matrix C on host with uniform-distribution random data
cutlass::reference::host::TensorFill(
tensor_d.host_view()); // <- fill matrix D on host with zeros
cutlass::reference::host::TensorFill(
tensor_ref_d.host_view()); // <- fill matrix D for reference on host with zeros
// Copy data from host to GPU
tensor_a.sync_device();
tensor_b.sync_device();
tensor_c.sync_device();
tensor_d.sync_device();
tensor_ref_d.sync_device();
// Initialize alpha and beta for dot product computation
ElementComputeEpilogue alpha = ElementComputeEpilogue(1);
ElementComputeEpilogue beta = ElementComputeEpilogue(0);
// Split K dimension into 16 partitions
int split_k_slices = 16;
// Create a tuple of gemm kernel arguments. This is later passed as arguments to launch
// instantiated CUTLASS kernel
typename Gemm::Arguments arguments{problem_size, // <- problem size of matrix multiplication
tensor_a.device_ref(), // <- reference to matrix A on device
tensor_b.device_ref(), // <- reference to matrix B on device
tensor_c.device_ref(), // <- reference to matrix C on device
tensor_d.device_ref(), // <- reference to matrix D on device
{alpha, beta}, // <- tuple of alpha and beta
split_k_slices}; // <- k-dimension split factor
// Using the arguments, query for extra workspace required for matrix multiplication computation
size_t workspace_size = Gemm::get_workspace_size(arguments);
// Allocate workspace memory
cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
// Instantiate CUTLASS kernel depending on templates
Gemm gemm_op;
// Initialize CUTLASS kernel with arguments and workspace pointer
cutlass::Status status = gemm_op.initialize(arguments, workspace.get());
CUTLASS_CHECK(status);
// Launch initialized CUTLASS kernel
status = gemm_op();
CUTLASS_CHECK(status);
// Create instantiation for device reference gemm kernel
cutlass::reference::device::Gemm<ElementInputA,
LayoutInputA,
ElementInputB,
LayoutInputB,
ElementOutput,
LayoutOutput,
ElementComputeEpilogue,
ElementComputeEpilogue>
gemm_device;
// Launch device reference gemm kernel
gemm_device(problem_size,
alpha,
tensor_a.device_ref(),
tensor_b.device_ref(),
beta,
tensor_c.device_ref(),
tensor_ref_d.device_ref());
// Wait for kernels to finish
cudaDeviceSynchronize();
// Copy output data from CUTLASS and reference kernel to host for comparison
tensor_d.sync_host();
tensor_ref_d.sync_host();
// Check if output from CUTLASS kernel and reference kernel are equal or not
std::cout << (cutlass::reference::host::TensorEquals(tensor_d.host_view(),
tensor_ref_d.host_view())
? "Passed"
: "Failed")
<< std::endl;
CUTLASS_CHECK(status);
}
examples/07_volta_tensorop_gemm/CMakeLists.txt
+27
View File
@@ -0,0 +1,27 @@
# Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are permitted
# provided that the following conditions are met:
# * Redistributions of source code must retain the above copyright notice, this list of
# conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice, this list of
# conditions and the following disclaimer in the documentation and/or other materials
# provided with the distribution.
# * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
# to endorse or promote products derived from this software without specific prior written
# permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
cutlass_example_add_executable(
07_volta_tensorop_gemm
volta_tensorop_gemm.cu
)
examples/07_volta_tensorop_gemm/volta_tensorop_gemm.cu
+323
View File
@@ -0,0 +1,323 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/**
This example shows how to run matrix multiplication kernels using functions and data structures
provided by CUTLASS using tensor cores; which we run on a NVIDIA Volta GPU.
Writing a single high performance matrix multiplication kernel is hard but do-able. Whereas writing
high performance kernels at scale which works for multiple problem sizes with good abstractions is
really hard. CUTLASS solves this problem by providing simplified abstractions (knobs) to compose
multiple sections of gemm kernel. When used properly, the kernels can hit peak performance of GPU
easily.
CUTLASS divides a kernel into hierarchical composable sections. Which means, at each thread, warp
and thread-block level, they compute on their own tile-size with higher level of tile sizes being
composed from lower level ones. Multiple thread-tiles (tile size each thread computes) can be used
to form warp-tiles (tile size each warp computes) and multiple warp tiles can be used to compute
threadblock-tile (tile size computed by a threadblock).
In thie example, we split variable initialization into
1. Setting up data properties : describes how matrices are laid out in the memory and how the kernel
can view them (logical to physical mapping)
2. Setting up computation properties : describes how the above set matrices will be used to compute
output of matrix multiplication.
First, we setup the data types of matrices A, B, C and D along with alpha, beta as the equation for
GEMM is D = alpha * A * B + beta * C. In CUTLASS, the kernels first compute A * B and leaves the
rest of the computation to end of the kernel as alpha * X + beta * C is a simple element-wise
operation on X (A * B) and C. We call this as epilogue of kernel. Hence, we setup data types for
alpha and beta to be equal to ElementComputeEpilogue = float. As we want to MMA instructions on
Volta and they support only half-precision floating point (fp16 or half), we use data type for
elements in input matrix A and B as cutlass::half_t. Volta also supports accumulation of partial dot
product to fp32, which can store wider range of numbers, we use it as data type of output matrix
elements and accumulation. We convey this to CUTLASS kernel by initializing template variables
ElementAccumulator (float), ElementComputeEpilogue (float), ElementInputA (cutlass::half_t),
ElementInputB (cutlass::half_t), ElementOutput (float). Communicating just the data type is not
enough. As the data is laid out linearly in memory, we have to convey the layout of matrices. We do
that by initializing template variable LayoutInputA to column major cutlass variable, LayoutInputB
to row major and LayoutOutput to row major. Next, we setup rules to comptue alpha * X + beta * C
which is called epilogue of the kernel. We initialize template variable EpilogueOp, which takes the
data type of output ElementOutput (int32_t), the number of elements per vector memory access (16),
data type of accumulator (int32_t) and data type of computation of linear combination (alpha * X +
beta * C).
Now that we setup the properties of data, we have to setup properties of computation.
Second, we create template variables of tile sizes for thread-block, warp and mma-op to 128x128x32,
64x64x4, 8x8x4 (MxNxK) respectively. When passed to instantiate CUTLASS GEMM kernel, it internally
deduce the amount of threads needed per thread-block, amount of shared memory, storing data in
bank-conflict free manner, and ton of other variables required to compose, intialize and launch a
high performance GEMM kernel. This is the beauty of CUTLASS, it relieves developer from
understanding and coding complicated hardware optimizations which can easily go wrong.
CUTLASS also supports multiple MMA pipelines in a CTA. What are MMA pipelines? MMA pipelines
constitute the whole process of loading input data from global memory to shared memory, loading data
from shared memory to registers, doing matrix multiplication, store to global memory. The below flow
sequence shows a typical mma pipeline.
matrix in global memory -> registers -> tile in shared memory -> registers -> mma -> registers ->
output to global memory
The problem with single pipeline is, each stage is synchronous which means, each stage has to wait
until the previous finished executing. There are stages in the pipeline which do not have fixed
latency, for example, the loads from global memory and shared memory. Therefore, we can add one more
pipeline with a phase shift in mma kernel to hide latency from global and shared memory loads.
Finally, the pipeline in a kernel looks like
(1) matrix in global memory -> (2) registers -> (3) tile in shared memory -> (4) registers -> (5)
mma -> (6) registers -> (7) output to global memory (1) <null> -> (2) <null> -> (3) matrix in global
memory -> (4) registers -> (5) tile in shared memory -> (6) registers -> (7) mma -> (8) registers ->
(9) output to global memory
This way, you can hide the second global memoroy load latency by doing computation on already loaded
input data.
There are few more template variables initialized such as, which threadblock tile of output matrix
is done which threadblock launched on an SM, CUDA SM architecture of GPU you want to run on.
These are all put together to create a template variable which describes CUTLASS GEMM kernel using
cutlass::gemm::device::Gemm template.
The next step is to intialize physical data, instantiate and initialize CUTLASS kernel and run it.
We use CUTLASS utilities to initialize, fill, compare matrices as they are simple and doesn't come
in the way of learning CUTLASS.
Once all the matrices are initialized and filled with data, create arguments tuple to launch CUTLASS
kernel which takes problem size (M = 5120, N = 4096 and K = 4096), matrices, alpha, beta and the
important one, split k-dimension factor. Along with that, we query CUTLASS if any scratch-space
memory required by the kernel we instantiated. If yes, we create it and pass it along with other
arguments created to intialize CUTLASS kernel then, the kernel is launched.
In this example, we later on launch a reference gemm kernel (from CUTLASS utilities) to compare if
the output from CUTLASS kernel is same as reference GEMM kernel.
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/device/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/tensor_view_io.h"
#include "helper.h"
// The code section below describes datatype for input, output matrices and computation between
// elements in input matrices.
using ElementAccumulator = float; // <- data type of accumulator
using ElementComputeEpilogue = ElementAccumulator; // <- data type of epilogue operations
using ElementInputA = cutlass::half_t; // <- data type of elements in input matrix A
using ElementInputB = cutlass::half_t; // <- data type of elements in input matrix B
using ElementOutput = float; // <- data type of elements in output matrix D
// The code section below describes matrix layout of input and output matrices. Column Major for
// Matrix A, Row Major for Matrix B and Row Major for Matrix C
using LayoutInputA = cutlass::layout::ColumnMajor;
using LayoutInputB = cutlass::layout::RowMajor;
using LayoutOutput = cutlass::layout::RowMajor;
// This code section describes whether you want to use tensor cores or regular SIMT cores on GPU SM
using MMAOp = cutlass::arch::OpClassTensorOp;
// This code section describes CUDA SM architecture number
using SmArch = cutlass::arch::Sm70;
// This code section describes the tile size a thread block will compute
using ShapeMMAThreadBlock =
cutlass::gemm::GemmShape<128, 128, 32>; // <- threadblock tile M = 128, N = 128, K = 32
// This code section describes tile size a warp will compute
using ShapeMMAWarp = cutlass::gemm::GemmShape<64, 64, 32>; // <- warp tile M = 64, N = 64, K = 32
// This code section describes the size of MMA op
using ShapeMMAOp = cutlass::gemm::GemmShape<8, 8, 4>; // <- MMA Op tile M = 8, N = 8, K = 4
// This code section describes how threadblocks are scheduled on GPU
using SwizzleThreadBlock = cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle; // <- ??
// This code section describes ?
using EpilogueOp = cutlass::epilogue::thread::LinearCombination<
ElementOutput, // <- data type of output matrix
128 / cutlass::sizeof_bits<ElementOutput>::value, // <- this is the number of elements per
// vectorized memory access. For half
// precision, it's 8 elements. This becomes
// the vector width of math instructions in
// epilogue too
ElementAccumulator, // <- data type of accumulator
ElementComputeEpilogue>; // <- data type for alpha/beta in linear combination function
// Number of pipelines you want to use
constexpr int NumStages = 2;
using Gemm = cutlass::gemm::device::Gemm<ElementInputA,
LayoutInputA,
ElementInputB,
LayoutInputB,
ElementOutput,
LayoutOutput,
ElementAccumulator,
MMAOp,
SmArch,
ShapeMMAThreadBlock,
ShapeMMAWarp,
ShapeMMAOp,
EpilogueOp,
SwizzleThreadBlock,
NumStages>;
int main() {
cudaDeviceProp props;
CUDA_CHECK(cudaGetDeviceProperties(&props, 0));
if (!(props.major >= 7)) {
std::cerr << "Volta Tensor Ops must be run on a machine with compute capability at least 70."
<< std::endl;
return 0;
}
const int length_m = 5120;
const int length_n = 4096;
const int length_k = 4096;
// Create a tuple of problem size for matrix multiplication
cutlass::gemm::GemmCoord problem_size(length_m, length_n, length_k);
// Initialize tensors using CUTLASS helper functions
cutlass::HostTensor<ElementInputA, LayoutInputA> tensor_a(
problem_size.mk()); // <- Create matrix A with dimensions M x K
cutlass::HostTensor<ElementInputB, LayoutInputB> tensor_b(
problem_size.nk()); // <- Create matrix B with dimensions N x K
cutlass::HostTensor<ElementOutput, LayoutOutput> tensor_c(
problem_size.mn()); // <- Create matrix C with dimensions M x N
cutlass::HostTensor<ElementOutput, LayoutOutput> tensor_d(
problem_size.mn()); // <- Create matrix D with dimensions M x N used to store output from
// CUTLASS kernel
cutlass::HostTensor<ElementOutput, LayoutOutput> tensor_ref_d(
problem_size.mn()); // <- Create matrix D with dimensions M x N used to store output from
// reference kernel
// Fill input and output matrices on host using CUTLASS helper functions
cutlass::reference::host::TensorFillRandomUniform(
tensor_a.host_view(),
1,
ElementInputA(4),
ElementInputA(-4),
0); // <- Fill matrix A on host with uniform-distribution random data
cutlass::reference::host::TensorFillRandomUniform(
tensor_b.host_view(),
1,
ElementInputB(4),
ElementInputB(-4),
0); // <- Fill matrix B on host with uniform-distribution random data
cutlass::reference::host::TensorFillRandomUniform(
tensor_c.host_view(),
1,
ElementOutput(4),
ElementOutput(-4),
0); // <- Fill matrix C on host with uniform-distribution random data
cutlass::reference::host::TensorFill(
tensor_d.host_view()); // <- fill matrix D on host with zeros
cutlass::reference::host::TensorFill(
tensor_ref_d.host_view()); // <- fill matrix D for reference on host with zeros
// Copy data from host to GPU
tensor_a.sync_device();
tensor_b.sync_device();
tensor_c.sync_device();
tensor_d.sync_device();
tensor_ref_d.sync_device();
// Initialize alpha and beta for dot product computation
ElementComputeEpilogue alpha = ElementComputeEpilogue(1);
ElementComputeEpilogue beta = ElementComputeEpilogue(0);
// Split K dimension into 1 partitions
int split_k_slices = 1;
// Create a tuple of gemm kernel arguments. This is later passed as arguments to launch
// instantiated CUTLASS kernel
typename Gemm::Arguments arguments{problem_size, // <- problem size of matrix multiplication
tensor_a.device_ref(), // <- reference to matrix A on device
tensor_b.device_ref(), // <- reference to matrix B on device
tensor_c.device_ref(), // <- reference to matrix C on device
tensor_d.device_ref(), // <- reference to matrix D on device
{alpha, beta}, // <- tuple of alpha and beta
split_k_slices}; // <- k-dimension split factor
// Using the arguments, query for extra workspace required for matrix multiplication computation
size_t workspace_size = Gemm::get_workspace_size(arguments);
// Allocate workspace memory
cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
// Instantiate CUTLASS kernel depending on templates
Gemm gemm_op;
// Initialize CUTLASS kernel with arguments and workspace pointer
cutlass::Status status = gemm_op.initialize(arguments, workspace.get());
CUTLASS_CHECK(status);
// Launch initialized CUTLASS kernel
status = gemm_op();
CUTLASS_CHECK(status);
// Create instantiation for device reference gemm kernel
cutlass::reference::device::Gemm<ElementInputA,
LayoutInputA,
ElementInputB,
LayoutInputB,
ElementOutput,
LayoutOutput,
ElementComputeEpilogue,
ElementComputeEpilogue>
gemm_device;
// Launch device reference gemm kernel
gemm_device(problem_size,
alpha,
tensor_a.device_ref(),
tensor_b.device_ref(),
beta,
tensor_c.device_ref(),
tensor_ref_d.device_ref());
// Wait for kernels to finish
cudaDeviceSynchronize();
// Copy output data from CUTLASS and reference kernel to host for comparison
tensor_d.sync_host();
tensor_ref_d.sync_host();
// Check if output from CUTLASS kernel and reference kernel are equal or not
std::cout << (cutlass::reference::host::TensorEquals(tensor_d.host_view(),
tensor_ref_d.host_view())
? "Passed"
: "Failed")
<< std::endl;
CUTLASS_CHECK(status);
return 0;
}
examples/08_turing_tensorop_gemm/CMakeLists.txt
+27
View File
@@ -0,0 +1,27 @@
# Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are permitted
# provided that the following conditions are met:
# * Redistributions of source code must retain the above copyright notice, this list of
# conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice, this list of
# conditions and the following disclaimer in the documentation and/or other materials
# provided with the distribution.
# * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
# to endorse or promote products derived from this software without specific prior written
# permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
cutlass_example_add_executable(
08_turing_tensorop_gemm
turing_tensorop_gemm.cu
)
examples/08_turing_tensorop_gemm/turing_tensorop_gemm.cu
+321
View File
@@ -0,0 +1,321 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/**
This example shows how to run matrix multiplication kernels using functions and data structures
provided by CUTLASS using tensor cores; which we run on a NVIDIA Turing GPU.
Writing a single high performance matrix multiplication kernel is hard but do-able. Whereas writing
high performance kernels at scale which works for multiple problem sizes with good abstractions is
really hard. CUTLASS solves this problem by providing simplified abstractions (knobs) to compose
multiple sections of gemm kernel. When used properly, the kernels can hit peak performance of GPU
easily.
CUTLASS divides a kernel into hierarchical composable sections. Which means, at each thread, warp
and thread-block level, they compute on their own tile-size with higher level of tile sizes being
composed from lower level ones. Multiple thread-tiles (tile size each thread computes) can be used
to form warp-tiles (tile size each warp computes) and multiple warp tiles can be used to compute
threadblock-tile (tile size computed by a threadblock).
In thie example, we split variable initialization into
1. Setting up data properties : describes how matrices are laid out in the memory and how the kernel
can view them (logical to physical mapping)
2. Setting up computation properties : describes how the above set matrices will be used to compute
output of matrix multiplication.
First, we setup the data types of matrices A, B, C and D along with alpha, beta as the equation for
GEMM is D = alpha * A * B + beta * C. In CUTLASS, the kernels first compute A * B and leaves the
rest of the computation to end of the kernel as alpha * X + beta * C is a simple element-wise
operation on X (A * B) and C. We call this as epilogue of kernel. Hence, we setup data types for
alpha and beta to be equal to ElementComputeEpilogue = int32_t. As we want to use MMA instructions
on Turing and they support 8-bit signed integer (int8_t), we use data type for elements in input
matrix A and B as int8_t. Volta also supports accumulation of partial dot product to int32_t, which
can store wider range of numbers, we use it as data type of output matrix elements and accumulation.
We convey this to CUTLASS kernel by initializing template variables ElementAccumulator (int32_t),
ElementComputeEpilogue (int32_t), ElementInputA (int8_t), ElementInputB (int8_t), ElementOutput
(int32_t). Communicating just the data type is not enough. As the data is laid out linearly in
memory, we have to convey the layout of matrices. We do that by initializing template variable
LayoutInputA to column major cutlass variable, LayoutInputB to row major and LayoutOutput to row
major. Next, we setup rules to comptue alpha * X + beta * C which is called epilogue of the kernel.
We initialize template variable EpilogueOp, which takes the data type of output ElementOutput
(int32_t), the number of elements per vector memory access (16), data type of accumulator (int32_t)
and data type of computation of linear combination (alpha * X + beta * C).
Now that we setup the properties of data, we have to setup properties of computation.
Second, we create template variables of tile sizes for thread-block, warp and mma-op to 128x256x64,
64x64x16, 8x8x16 (MxNxK) respectively. When passed to instantiate CUTLASS GEMM kernel, it internally
deduce the amount of threads needed per thread-block, amount of shared memory, storing data in
bank-conflict free manner, and ton of other variables required to compose, intialize and launch a
high performance GEMM kernel. This is the beauty of CUTLASS, it relieves developer from
understanding and coding complicated hardware optimizations which can easily go wrong.
CUTLASS also supports multiple MMA pipelines in a threadblock. What are MMA pipelines? MMA pipelines
constitute the whole process of loading input data from global memory to shared memory, loading data
from shared memory to registers, doing matrix multiplication, store to global memory. The below flow
sequence shows a typical mma pipeline.
matrix in global memory -> registers -> tile in shared memory -> registers -> mma -> registers ->
output to global memory
The problem with single pipeline is, each stage is synchronous which means, each stage has to wait
until the previous finished executing. There are stages in the pipeline which do not have fixed
latency, for example, the loads from global memory and shared memory. Therefore, we can add one more
pipeline with a phase shift in mma kernel to hide latency from global and shared memory loads.
Finally, the pipeline in a kernel looks like
(1) matrix in global memory -> (2) registers -> (3) tile in shared memory -> (4) registers -> (5)
mma -> (6) registers -> (7) output to global memory (1) <null> -> (2) <null> -> (3) matrix in global
memory -> (4) registers -> (5) tile in shared memory -> (6) registers -> (7) mma -> (8) registers ->
(9) output to global memory
This way, you can hide the second global memoroy load latency by doing computation on already loaded
input data.
There are few more template variables initialized such as, which threadblock tile of output matrix
is done which threadblock launched on an SM, CUDA SM architecture of GPU you want to run on.
These are all put together to create a template variable which describes CUTLASS GEMM kernel using
cutlass::gemm::device::Gemm template.
The next step is to intialize physical data, instantiate and initialize CUTLASS kernel and run it.
We use CUTLASS utilities to initialize, fill, compare matrices as they are simple and doesn't come
in the way of learning CUTLASS.
Once all the matrices are initialized and filled with data, create arguments tuple to launch CUTLASS
kernel which takes problem size (M = 5120, N = 4096 and K = 4096), matrices, alpha, beta and the
important one, split k-dimension factor. Along with that, we query CUTLASS if any scratch-space
memory required by the kernel we instantiated. If yes, we create it and pass it along with other
arguments created to intialize CUTLASS kernel then, the kernel is launched.
In this example, we later on launch a reference gemm kernel (from CUTLASS utilities) to compare if
the output from CUTLASS kernel is same as reference GEMM kernel.
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/device/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/tensor_view_io.h"
#include "helper.h"
// The code section below describes datatype for input, output matrices and computation between
// elements in input matrices.
using ElementAccumulator = int32_t; // <- data type of accumulator
using ElementComputeEpilogue = ElementAccumulator; // <- data type of epilogue operations
using ElementInputA = int8_t; // <- data type of elements in input matrix A
using ElementInputB = int8_t; // <- data type of elements in input matrix B
using ElementOutput = int32_t; // <- data type of elements in output matrix D
// The code section below describes matrix layout of input and output matrices. Column Major for
// Matrix A, Row Major for Matrix B and Row Major for Matrix C
using LayoutInputA = cutlass::layout::RowMajor;
using LayoutInputB = cutlass::layout::ColumnMajor;
using LayoutOutput = cutlass::layout::RowMajor;
// This code section describes whether you want to use tensor cores or regular SIMT cores on GPU SM
using MMAOp = cutlass::arch::OpClassTensorOp;
// This code section describes CUDA SM architecture number
using SmArch = cutlass::arch::Sm75;
// This code section describes the tile size a thread block will compute
using ShapeMMAThreadBlock =
cutlass::gemm::GemmShape<128, 256, 64>; // <- threadblock tile M = 128, N = 256, K = 64
// This code section describes tile size a warp will compute
using ShapeMMAWarp = cutlass::gemm::GemmShape<64, 64, 64>; // <- warp tile M = 64, N = 64, K = 16
// This code section describes the size of MMA op
using ShapeMMAOp = cutlass::gemm::GemmShape<8, 8, 16>; // <- MMA Op tile M = 8, N = 8, K = 16
// This code section describes how threadblocks are scheduled on GPU
using SwizzleThreadBlock = cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle; // <- ??
// This code section describes the epilogue part of the kernel
using EpilogueOp = cutlass::epilogue::thread::LinearCombination<
ElementOutput, // <- data type of output matrix
128 / cutlass::sizeof_bits<ElementOutput>::value, // <- the number of elements per vectorized
// memory access. For a byte, it's 16
// elements. This becomes the vector width of
// math instructions in the epilogue too
ElementAccumulator, // <- data type of accumulator
ElementComputeEpilogue>; // <- data type for alpha/beta in linear combination function
// Number of pipelines you want to use
constexpr int NumStages = 2;
using Gemm = cutlass::gemm::device::Gemm<ElementInputA,
LayoutInputA,
ElementInputB,
LayoutInputB,
ElementOutput,
LayoutOutput,
ElementAccumulator,
MMAOp,
SmArch,
ShapeMMAThreadBlock,
ShapeMMAWarp,
ShapeMMAOp,
EpilogueOp,
SwizzleThreadBlock,
NumStages>;
int main() {
cudaDeviceProp props;
CUDA_CHECK(cudaGetDeviceProperties(&props, 0));
if (!(props.major >= 7 && props.minor >= 5)) {
std::cerr << "Turing Tensor Ops must be run on a machine with compute capability at least 75."
<< std::endl;
return 0;
}
const int length_m = 5120;
const int length_n = 4096;
const int length_k = 4096;
// Create a tuple of problem size for matrix multiplication
cutlass::gemm::GemmCoord problem_size(length_m, length_n, length_k);
// Initialize tensors using CUTLASS helper functions
cutlass::HostTensor<ElementInputA, LayoutInputA> tensor_a(
problem_size.mk()); // <- Create matrix A with dimensions M x K
cutlass::HostTensor<ElementInputB, LayoutInputB> tensor_b(
problem_size.nk()); // <- Create matrix B with dimensions N x K
cutlass::HostTensor<ElementOutput, LayoutOutput> tensor_c(
problem_size.mn()); // <- Create matrix C with dimensions M x N
cutlass::HostTensor<ElementOutput, LayoutOutput> tensor_d(
problem_size.mn()); // <- Create matrix D with dimensions M x N used to store output from
// CUTLASS kernel
cutlass::HostTensor<ElementOutput, LayoutOutput> tensor_ref_d(
problem_size.mn()); // <- Create matrix D with dimensions M x N used to store output from
// reference kernel
// Fill input and output matrices on host using CUTLASS helper functions
cutlass::reference::host::TensorFillRandomUniform(
tensor_a.host_view(),
1,
ElementInputA(4),
ElementInputA(-4),
0); // <- Fill matrix A on host with uniform-distribution random data
cutlass::reference::host::TensorFillRandomUniform(
tensor_b.host_view(),
1,
ElementInputB(4),
ElementInputB(-4),
0); // <- Fill matrix B on host with uniform-distribution random data
cutlass::reference::host::TensorFillRandomUniform(
tensor_c.host_view(),
1,
ElementOutput(4),
ElementOutput(-4),
0); // <- Fill matrix C on host with uniform-distribution random data
cutlass::reference::host::TensorFill(
tensor_d.host_view()); // <- fill matrix D on host with zeros
cutlass::reference::host::TensorFill(
tensor_ref_d.host_view()); // <- fill matrix D for reference on host with zeros
// Copy data from host to GPU
tensor_a.sync_device();
tensor_b.sync_device();
tensor_c.sync_device();
tensor_d.sync_device();
tensor_ref_d.sync_device();
// Initialize alpha and beta for dot product computation
ElementComputeEpilogue alpha = ElementComputeEpilogue(1);
ElementComputeEpilogue beta = ElementComputeEpilogue(0);
// Split K dimension into 1 partitions
int split_k_slices = 1;
// Create a tuple of gemm kernel arguments. This is later passed as arguments to launch
// instantiated CUTLASS kernel
typename Gemm::Arguments arguments{problem_size, // <- problem size of matrix multiplication
tensor_a.device_ref(), // <- reference to matrix A on device
tensor_b.device_ref(), // <- reference to matrix B on device
tensor_c.device_ref(), // <- reference to matrix C on device
tensor_d.device_ref(), // <- reference to matrix D on device
{alpha, beta}, // <- tuple of alpha and beta
split_k_slices}; // <- k-dimension split factor
// Using the arguments, query for extra workspace required for matrix multiplication computation
size_t workspace_size = Gemm::get_workspace_size(arguments);
// Allocate workspace memory
cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
// Instantiate CUTLASS kernel depending on templates
Gemm gemm_op;
// Initialize CUTLASS kernel with arguments and workspace pointer
cutlass::Status status = gemm_op.initialize(arguments, workspace.get());
CUTLASS_CHECK(status);
// Launch initialized CUTLASS kernel
status = gemm_op();
CUTLASS_CHECK(status);
// Create instantiation for device reference gemm kernel
cutlass::reference::device::Gemm<ElementInputA,
LayoutInputA,
ElementInputB,
LayoutInputB,
ElementOutput,
LayoutOutput,
ElementComputeEpilogue,
ElementComputeEpilogue>
gemm_device;
// Launch device reference gemm kernel
gemm_device(problem_size,
alpha,
tensor_a.device_ref(),
tensor_b.device_ref(),
beta,
tensor_c.device_ref(),
tensor_ref_d.device_ref());
// Wait for kernels to finish
cudaDeviceSynchronize();
// Copy output data from CUTLASS and reference kernel to host for comparison
tensor_d.sync_host();
tensor_ref_d.sync_host();
// Check if output from CUTLASS kernel and reference kernel are equal or not
std::cout << (cutlass::reference::host::TensorEquals(tensor_d.host_view(),
tensor_ref_d.host_view())
? "Passed"
: "Failed")
<< std::endl;
CUTLASS_CHECK(status);
return 0;
}
examples/CMakeLists.txt
+44 -7
View File
@@ -20,11 +20,48 @@
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
add_subdirectory(00_basic_gemm)
add_subdirectory(01_tensor_view)
add_subdirectory(02_cutlass_utilities)
add_subdirectory(03_strided_batched_gemm)
add_subdirectory(04_tile_iterator)
add_subdirectory(05_wmma_gemm)
add_subdirectory(06_splitK_gemm)
set(CUTLASS_EXAMPLES_COMMON_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/common)
function(cutlass_example_add_executable)
set(options)
set(oneValueArgs)
set(multiValueArgs)
cmake_parse_arguments(_ "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
cutlass_add_executable(${__UNPARSED_ARGUMENTS})
list(GET __UNPARSED_ARGUMENTS 0 NAME)
target_link_libraries(
${NAME}
PRIVATE
CUTLASS
cutlass_tools_util_includes
)
target_include_directories(
${NAME}
PRIVATE
${CUTLASS_EXAMPLES_COMMON_SOURCE_DIR}
)
endfunction()
add_custom_target(cutlass_examples)
foreach(EXAMPLE
00_basic_gemm
01_cutlass_utilities
02_dump_reg_shmem
03_visualize_layout
04_tile_iterator
05_batched_gemm
06_splitK_gemm
07_volta_tensorop_gemm
08_turing_tensorop_gemm)
add_subdirectory(${EXAMPLE})
add_dependencies(cutlass_examples ${EXAMPLE})
endforeach()
examples/common/helper.h
+23
View File
@@ -0,0 +1,23 @@
#pragma once
#include "cuda_runtime.h"
#define CUTLASS_CHECK(status) \
{ \
cutlass::Status error = status; \
if (error != cutlass::Status::kSuccess) { \
std::cerr << "Got cutlass error: " << cutlassGetStatusString(error) << " at: " << __LINE__ \
<< std::endl; \
exit(EXIT_FAILURE); \
} \
}
#define CUDA_CHECK(status) \
{ \
cudaError_t error = status; \
if (error != cudaSuccess) { \
std::cerr << "Got bad cuda status: " << cudaGetErrorString(error) \
<< " at line: " << __LINE__ << std::endl; \
exit(EXIT_FAILURE); \
} \
}