Checkpointing CUTLASS 1.1 release.
This commit is contained in:
akerr
@@ -0,0 +1,38 @@
|
||||
# Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
# provided that the following conditions are met:
|
||||
# * Redistributions of source code must retain the above copyright notice, this list of
|
||||
# conditions and the following disclaimer.
|
||||
# * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
# conditions and the following disclaimer in the documentation and/or other materials
|
||||
# provided with the distribution.
|
||||
# * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
# to endorse or promote products derived from this software without specific prior written
|
||||
# permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
set(EXAMPLES_BASIC_CUTLASS_GEMM_SOURCES
|
||||
basic_gemm.cu
|
||||
)
|
||||
|
||||
if (NOT CUTLASS_NATIVE_CUDA)
|
||||
# cuda_add_executable does not take interface include directories into account
|
||||
# Let's fetch them and pass them to CUDA.
|
||||
get_target_property(CUTLASS_INCLUDES CUTLASS INTERFACE_INCLUDE_DIRECTORIES)
|
||||
include_directories("${CUTLASS_INCLUDES}")
|
||||
endif()
|
||||
|
||||
cutlass_add_executable(
|
||||
00_basic_gemm
|
||||
${EXAMPLES_BASIC_CUTLASS_GEMM_SOURCES}
|
||||
)
|
||||
@@ -0,0 +1,492 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*
|
||||
This example demonstrates how to call a CUTLASS GEMM kernel and provides a naive reference
|
||||
matrix multiply kernel to verify its correctness.
|
||||
|
||||
The CUTLASS Gemm template is instantiated in the function CutlassSgemmNN. This is kernel computes
|
||||
the general matrix product (GEMM) using single-precision floating-point arithmetic and assumes
|
||||
all matrices have column-major layout.
|
||||
|
||||
The threadblock tile size is chosen as 128x128x8 which offers good performance for large matrices.
|
||||
See the CUTLASS Parallel for All blog post for more exposition on the tunable parameters available
|
||||
in CUTLASS.
|
||||
|
||||
https://devblogs.nvidia.com/cutlass-linear-algebra-cuda/
|
||||
|
||||
Aside from defining and launching the SGEMM kernel, this example does not use any other components
|
||||
or utilities within CUTLASS. Such utilities are demonstrated elsewhere in other examples and are
|
||||
prevalent in the CUTLASS unit tests.
|
||||
*/
|
||||
|
||||
// Standard Library includes
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <vector>
|
||||
|
||||
//
|
||||
// CUTLASS includes needed for single-precision GEMM kernel
|
||||
//
|
||||
|
||||
// Defines cutlass::gemm::Gemm, the generic Gemm computation template class.
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
|
||||
// Defines cutlass::gemm::SgemmTraits, the structural components for single-precision GEMM
|
||||
#include "cutlass/gemm/sgemm_traits.h"
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// This function defines a CUTLASS GEMM kernel instantiation, constructs its parameters object,
|
||||
// and launches it on the CUDA device.
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Define a CUTLASS GEMM template and launch a GEMM kernel.
|
||||
cudaError_t CutlassSgemmNN(
|
||||
int M,
|
||||
int N,
|
||||
int K,
|
||||
float alpha,
|
||||
float const *A,
|
||||
int lda,
|
||||
float const *B,
|
||||
int ldb,
|
||||
float beta,
|
||||
float *C,
|
||||
int ldc) {
|
||||
|
||||
// Define type definition for single-precision CUTLASS GEMM with column-major
|
||||
// input matrices and 128x128x8 threadblock tile size.
|
||||
//
|
||||
// Note, GemmTraits<> is a generic template defined for various general matrix product
|
||||
// computations within CUTLASS. It is intended to be maximally flexible, and consequently
|
||||
// it contains numerous template arguments.
|
||||
//
|
||||
// To keep the interface manageable, several helpers are defined for plausible compositions
|
||||
// including the following example for single-precision GEMM. Typical values are used as
|
||||
// default template arguments. See `cutlass/gemm/gemm_traits.h` for more details.
|
||||
//
|
||||
typedef cutlass::gemm::SgemmTraits<
|
||||
cutlass::MatrixLayout::kColumnMajor, // layout of A matrix
|
||||
cutlass::MatrixLayout::kColumnMajor, // layout of B matrix
|
||||
cutlass::Shape<8, 128, 128> // threadblock tile size
|
||||
>
|
||||
GemmTraits;
|
||||
|
||||
// Define a CUTLASS GEMM type from a GemmTraits<> instantiation.
|
||||
typedef cutlass::gemm::Gemm<GemmTraits> Gemm;
|
||||
|
||||
// Construct and initialize CUTLASS GEMM parameters object.
|
||||
//
|
||||
// One of CUTLASS's design patterns is to define parameters objects that are constructible
|
||||
// in host code and passed to kernels by value. These may include pointers, strides, scalars,
|
||||
// and other arguments needed by Gemm and its components.
|
||||
//
|
||||
// The benefits of this pattern are (1.) a structured, composable strategy for passing host-constructible
|
||||
// arguments to kernels and (2.) minimized initialization overhead on kernel entry.
|
||||
//
|
||||
typename Gemm::Params params;
|
||||
|
||||
int result = params.initialize(
|
||||
M, // GEMM M dimension
|
||||
N, // GEMM N dimension
|
||||
K, // GEMM K dimension
|
||||
alpha, // scalar alpha
|
||||
A, // matrix A operand
|
||||
lda,
|
||||
B, // matrix B operand
|
||||
ldb,
|
||||
beta, // scalar beta
|
||||
C, // source matrix C
|
||||
ldc,
|
||||
C, // destination matrix C (may be different memory than source C matrix)
|
||||
ldc
|
||||
);
|
||||
|
||||
if (result) {
|
||||
std::cerr << "Failed to initialize CUTLASS Gemm::Params object." << std::endl;
|
||||
return cudaErrorInvalidValue;
|
||||
}
|
||||
|
||||
// Launch the CUTLASS GEMM kernel.
|
||||
Gemm::launch(params);
|
||||
|
||||
// Return any errors associated with the launch or cudaSuccess if no error.
|
||||
return cudaGetLastError();
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The source code after this point in the file is generic CUDA using the CUDA Runtime API
|
||||
// and simple CUDA kernels to initialize matrices and compute the general matrix product.
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Kernel to initialize a matrix with small integers.
|
||||
__global__ void InitializeMatrix_kernel(
|
||||
float *matrix,
|
||||
int ldm,
|
||||
int rows,
|
||||
int columns,
|
||||
int seed = 0) {
|
||||
|
||||
int i = threadIdx.x + blockIdx.x * blockDim.x;
|
||||
int j = threadIdx.y + blockIdx.y * blockDim.y;
|
||||
|
||||
if (i < rows && j < columns) {
|
||||
int offset = i + j * ldm;
|
||||
|
||||
// Generate arbitrary elements.
|
||||
int const k = 16807;
|
||||
int const m = 16;
|
||||
float value = float(((offset + seed) * k % m) - m / 2);
|
||||
|
||||
matrix[offset] = value;
|
||||
}
|
||||
}
|
||||
|
||||
/// Simple function to initialize a matrix to arbitrary small integers.
|
||||
cudaError_t InitializeMatrix(float *matrix, int ldm, int rows, int columns, int seed = 0) {
|
||||
|
||||
dim3 block(16, 16);
|
||||
dim3 grid(
|
||||
(rows + block.x - 1) / block.x,
|
||||
(columns + block.y - 1) / block.y
|
||||
);
|
||||
|
||||
InitializeMatrix_kernel<<< grid, block >>>(matrix, ldm, rows, columns, seed);
|
||||
|
||||
return cudaGetLastError();
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Allocates device memory for a matrix then fills with arbitrary small integers.
|
||||
cudaError_t AllocateMatrix(float **matrix, int ldm, int rows, int columns, int seed = 0) {
|
||||
cudaError_t result;
|
||||
|
||||
size_t sizeof_matrix = sizeof(float) * ldm * columns;
|
||||
|
||||
// Allocate device memory.
|
||||
result = cudaMalloc(reinterpret_cast<void **>(matrix), sizeof_matrix);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Failed to allocate matrix: "
|
||||
<< cudaGetErrorString(result) << std::endl;
|
||||
return result;
|
||||
}
|
||||
|
||||
// Clear the allocation.
|
||||
result = cudaMemset(*matrix, 0, sizeof_matrix);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Failed to clear matrix device memory: "
|
||||
<< cudaGetErrorString(result) << std::endl;
|
||||
return result;
|
||||
}
|
||||
|
||||
// Initialize matrix elements to arbitrary small integers.
|
||||
result = InitializeMatrix(*matrix, ldm, rows, columns, seed);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Failed to initialize matrix: "
|
||||
<< cudaGetErrorString(result) << std::endl;
|
||||
return result;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Naive reference GEMM computation.
|
||||
__global__ void ReferenceGemm_kernel(
|
||||
int M,
|
||||
int N,
|
||||
int K,
|
||||
float alpha,
|
||||
float const *A,
|
||||
int lda,
|
||||
float const *B,
|
||||
int ldb,
|
||||
float beta,
|
||||
float *C,
|
||||
int ldc) {
|
||||
|
||||
int i = threadIdx.x + blockIdx.x * blockDim.x;
|
||||
int j = threadIdx.y + blockIdx.y * blockDim.y;
|
||||
|
||||
if (i < M && j < N) {
|
||||
float accumulator = 0;
|
||||
|
||||
for (int k = 0; k < K; ++k) {
|
||||
accumulator += A[i + k * lda] * B[k + j * ldb];
|
||||
}
|
||||
|
||||
C[i + j * ldc] = alpha * accumulator + beta * C[i + j * ldc];
|
||||
}
|
||||
}
|
||||
|
||||
/// Reference GEMM computation.
|
||||
cudaError_t ReferenceGemm(
|
||||
int M,
|
||||
int N,
|
||||
int K,
|
||||
float alpha,
|
||||
float const *A,
|
||||
int lda,
|
||||
float const *B,
|
||||
int ldb,
|
||||
float beta,
|
||||
float *C,
|
||||
int ldc) {
|
||||
|
||||
dim3 block(16, 16);
|
||||
dim3 grid(
|
||||
(M + block.x - 1) / block.x,
|
||||
(N + block.y - 1) / block.y
|
||||
);
|
||||
|
||||
ReferenceGemm_kernel<<< grid, block >>>(M, N, K, alpha, A, lda, B, ldb, beta, C, ldc);
|
||||
|
||||
return cudaGetLastError();
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Allocate several matrices in GPU device memory and call a single-precision
|
||||
/// CUTLASS GEMM kernel.
|
||||
cudaError_t TestCutlassGemm(int M, int N, int K, float alpha, float beta) {
|
||||
cudaError_t result;
|
||||
|
||||
//
|
||||
// Define several matrices to be used as operands to GEMM kernels.
|
||||
//
|
||||
|
||||
// Compute leading dimensions for each matrix.
|
||||
int lda = M;
|
||||
int ldb = K;
|
||||
int ldc = M;
|
||||
|
||||
// Compute size in bytes of the C matrix.
|
||||
size_t sizeof_C = sizeof(float) * ldc * N;
|
||||
|
||||
// Define pointers to matrices in GPU device memory.
|
||||
float *A;
|
||||
float *B;
|
||||
float *C_cutlass;
|
||||
float *C_reference;
|
||||
|
||||
//
|
||||
// Allocate matrices in GPU device memory with arbitrary seeds.
|
||||
//
|
||||
|
||||
result = AllocateMatrix(&A, lda, M, K, 0);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
return result;
|
||||
}
|
||||
|
||||
result = AllocateMatrix(&B, ldb, K, N, 17);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
cudaFree(A);
|
||||
return result;
|
||||
}
|
||||
|
||||
result = AllocateMatrix(&C_cutlass, ldc, M, N, 101);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
cudaFree(A);
|
||||
cudaFree(B);
|
||||
return result;
|
||||
}
|
||||
|
||||
result = AllocateMatrix(&C_reference, ldc, M, N, 101);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
cudaFree(A);
|
||||
cudaFree(B);
|
||||
cudaFree(C_cutlass);
|
||||
return result;
|
||||
}
|
||||
|
||||
result = cudaMemcpy(C_reference, C_cutlass, sizeof_C, cudaMemcpyDeviceToDevice);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Failed to copy C_cutlass matrix to C_reference: "
|
||||
<< cudaGetErrorString(result) << std::endl;
|
||||
|
||||
cudaFree(C_reference);
|
||||
cudaFree(C_cutlass);
|
||||
cudaFree(B);
|
||||
cudaFree(A);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
//
|
||||
// Launch CUTLASS GEMM.
|
||||
//
|
||||
|
||||
result = CutlassSgemmNN(M, N, K, alpha, A, lda, B, ldb, beta, C_cutlass, ldc);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "CUTLASS GEMM kernel failed: "
|
||||
<< cudaGetErrorString(result) << std::endl;
|
||||
|
||||
cudaFree(C_reference);
|
||||
cudaFree(C_cutlass);
|
||||
cudaFree(B);
|
||||
cudaFree(A);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
//
|
||||
// Verify.
|
||||
//
|
||||
|
||||
// Launch reference GEMM
|
||||
result = ReferenceGemm(M, N, K, alpha, A, lda, B, ldb, beta, C_reference, ldc);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Reference GEMM kernel failed: "
|
||||
<< cudaGetErrorString(result) << std::endl;
|
||||
|
||||
cudaFree(C_reference);
|
||||
cudaFree(C_cutlass);
|
||||
cudaFree(B);
|
||||
cudaFree(A);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// Copy to host and verify equivalence.
|
||||
std::vector<float> host_cutlass(ldc * N, 0);
|
||||
std::vector<float> host_reference(ldc * N, 0);
|
||||
|
||||
result = cudaMemcpy(host_cutlass.data(), C_cutlass, sizeof_C, cudaMemcpyDeviceToHost);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Failed to copy CUTLASS GEMM results: "
|
||||
<< cudaGetErrorString(result) << std::endl;
|
||||
|
||||
cudaFree(C_reference);
|
||||
cudaFree(C_cutlass);
|
||||
cudaFree(B);
|
||||
cudaFree(A);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
result = cudaMemcpy(host_reference.data(), C_reference, sizeof_C, cudaMemcpyDeviceToHost);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Failed to copy Reference GEMM results: "
|
||||
<< cudaGetErrorString(result) << std::endl;
|
||||
|
||||
cudaFree(C_reference);
|
||||
cudaFree(C_cutlass);
|
||||
cudaFree(B);
|
||||
cudaFree(A);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
//
|
||||
// Free device memory allocations.
|
||||
//
|
||||
|
||||
cudaFree(C_reference);
|
||||
cudaFree(C_cutlass);
|
||||
cudaFree(B);
|
||||
cudaFree(A);
|
||||
|
||||
//
|
||||
// Test for bit equivalence of results.
|
||||
//
|
||||
|
||||
if (host_cutlass != host_reference) {
|
||||
std::cerr << "CUTLASS results incorrect." << std::endl;
|
||||
|
||||
return cudaErrorUnknown;
|
||||
}
|
||||
|
||||
return cudaSuccess;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Entry point to basic_gemm example.
|
||||
//
|
||||
// usage:
|
||||
//
|
||||
// 00_basic_gemm <M> <N> <K> <alpha> <beta>
|
||||
//
|
||||
int main(int argc, const char *arg[]) {
|
||||
|
||||
//
|
||||
// Parse the command line to obtain GEMM dimensions and scalar values.
|
||||
//
|
||||
|
||||
// GEMM problem dimensions.
|
||||
int problem[3] = { 128, 128, 128 };
|
||||
|
||||
for (int i = 1; i < argc && i < 4; ++i) {
|
||||
std::stringstream ss(arg[i]);
|
||||
ss >> problem[i - 1];
|
||||
}
|
||||
|
||||
// Scalars used for linear scaling the result of the matrix product.
|
||||
float scalars[2] = { 1, 0 };
|
||||
|
||||
for (int i = 4; i < argc && i < 6; ++i) {
|
||||
std::stringstream ss(arg[i]);
|
||||
ss >> scalars[i - 4];
|
||||
}
|
||||
|
||||
//
|
||||
// Run the CUTLASS GEMM test.
|
||||
//
|
||||
|
||||
cudaError_t result = TestCutlassGemm(
|
||||
problem[0], // GEMM M dimension
|
||||
problem[1], // GEMM N dimension
|
||||
problem[2], // GEMM K dimension
|
||||
scalars[0], // alpha
|
||||
scalars[1] // beta
|
||||
);
|
||||
|
||||
if (result == cudaSuccess) {
|
||||
std::cout << "Passed." << std::endl;
|
||||
}
|
||||
|
||||
// Exit.
|
||||
return result == cudaSuccess ? 0 : -1;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,38 @@
|
||||
# Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
# provided that the following conditions are met:
|
||||
# * Redistributions of source code must retain the above copyright notice, this list of
|
||||
# conditions and the following disclaimer.
|
||||
# * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
# conditions and the following disclaimer in the documentation and/or other materials
|
||||
# provided with the distribution.
|
||||
# * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
# to endorse or promote products derived from this software without specific prior written
|
||||
# permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
set(EXAMPLES_TENSOR_VIEW_SOURCES
|
||||
tensor_view.cu
|
||||
)
|
||||
|
||||
if (NOT CUTLASS_NATIVE_CUDA)
|
||||
# cuda_add_executable does not take interface include directories into account
|
||||
# Let's fetch them and pass them to CUDA.
|
||||
get_target_property(CUTLASS_INCLUDES CUTLASS INTERFACE_INCLUDE_DIRECTORIES)
|
||||
include_directories("${CUTLASS_INCLUDES}")
|
||||
endif()
|
||||
|
||||
cutlass_add_executable(
|
||||
01_tensor_view
|
||||
${EXAMPLES_TENSOR_VIEW_SOURCES}
|
||||
)
|
||||
@@ -0,0 +1,424 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*
|
||||
This example demonstrates operations using TensorRef<> and TensorView<> as well as their explicit
|
||||
equivalent functionality in CUDA code.
|
||||
|
||||
CUTLASS provides abstractions for interacting with multidimension tensors in device memory.
|
||||
Consequently, we define a hierarchy of pointer-like types for referencing tensors.
|
||||
|
||||
T * - raw pointer to elements of type T
|
||||
|
||||
cutlass::TensorRef<T, Rank> - reference to a tensor of elements of type T and given rank.
|
||||
Includes a mapping function and associated stride vector for
|
||||
accessing elements in linear memory.
|
||||
|
||||
cutlass::TensorView<T, Rank>: - extends TensorRef<> by adding bounds information. This is a
|
||||
public TensorRef<T, Rank> complete mathematical object which may be used as the argument
|
||||
to CUTLASS functions.
|
||||
|
||||
The above provide an identity maping of a logical index space to linear memory. An element
|
||||
at logical coordinate X has an offset computed as follows:
|
||||
|
||||
offset = dot(X, stride)
|
||||
|
||||
where dot() computes the inner product of X and a vector of "strides."
|
||||
|
||||
CUTLASS 1.1 introduces a mapping function and an additional 'rank' to offer a flexible way to
|
||||
map the logical index space of the tensor to memory. The mapping function maps a coordinate
|
||||
of rank R to an index space of rank S. The linear offset is computed as:
|
||||
|
||||
offset = dot( MapFunc(X), stride )
|
||||
|
||||
where stride is a vector of rank S.
|
||||
|
||||
|
||||
The complete template declaration for cutlass::TensorRef<> is as follows.
|
||||
|
||||
template <
|
||||
/// Data type of element stored within tensor
|
||||
typename Storage,
|
||||
|
||||
/// Rank of logical tensor
|
||||
int Rank,
|
||||
|
||||
/// Maps a Coord<Rank> in the logical tensor index space to the internal n-D array
|
||||
typename MapFunc = IdentityTensorMapFunc<Rank>,
|
||||
|
||||
/// Rank of internal n-D array
|
||||
int StorageRank_ = MapFunc::kStorageRank,
|
||||
|
||||
/// Index type used for coordinates
|
||||
typename Index = int,
|
||||
|
||||
/// Index type used for offsets and pointer differences
|
||||
typename LongIndex = long long
|
||||
>
|
||||
class TensorRef;
|
||||
|
||||
|
||||
CUTLASS kernels make extensive use of vectorization of memory accesses for efficiency and
|
||||
correctness. Consequently, we enforce a constraint on the strides used by mapping functions
|
||||
such that:
|
||||
|
||||
1. The "fastest-changing" stride is always 1 thereby mandating that consecutive elements in
|
||||
that rank are consecutive in linear memory.
|
||||
|
||||
2. The fastest changing rank is always last in the stride vector and not explicitly stored.
|
||||
|
||||
Thus, the stride vector used by mapping functions has length of one fewer than the rank of the
|
||||
storage tensor. These constraints are consistent with the BLAS interface of passing matrices as
|
||||
a tuple consisting of a pointer and a "leading dimension." In fact, these are rank=2 tensors
|
||||
whose fastest changing dimension is 1, and the stride vector is of length 1.
|
||||
|
||||
|
||||
A typical mapping function might simply map the rows and columns of a matrix, a rank=2 tensor,
|
||||
to linear memory such that (1.) elements in the same column are consecutive in memory
|
||||
(column-major), or (2.) elements in the same row are consecutive (row-major). These can be
|
||||
accomplished by two different mapping functions whose stride vector is length=2. The first
|
||||
element is the "leading dimension."
|
||||
|
||||
The following mapping functions demonstrates mappings for these canonical matrix layouts. In
|
||||
both cases, the logical index space is referenced by coordinates of the form (row, column).
|
||||
|
||||
// cutlass/matrix_traits.h
|
||||
struct MatrixLayout {
|
||||
|
||||
//
|
||||
// TensorRefMapFunc definitions for common layouts
|
||||
//
|
||||
|
||||
/// Mapping function for row-major matrices
|
||||
struct RowMajor {
|
||||
|
||||
/// Storage rank = 2 implies stride vector: (ldm, 1)
|
||||
static int const kStorageRank = 2;
|
||||
|
||||
/// Maps (row, col) to (row, col)
|
||||
CUTLASS_HOST_DEVICE
|
||||
Coord<kStorageRank> operator()(Coord<2> const &coord) const {
|
||||
return coord;
|
||||
}
|
||||
};
|
||||
|
||||
/// Mapping function for column-major matrices
|
||||
struct ColumnMajor {
|
||||
|
||||
/// Storage rank = 2 implies stride vector: (ldm, 1)
|
||||
static int const kStorageRank = 2;
|
||||
|
||||
/// Maps (row, col) to (col, row)
|
||||
CUTLASS_HOST_DEVICE
|
||||
Coord<kStorageRank> operator()(Coord<2> const &coord) const {
|
||||
return make_Coord(coord[1], coord[0]);
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
The requirement that the fastest-changing stride always be of unit size need not be a limitation.
|
||||
To implement "sparse" computations or matrix operations in which matrix elements have arbitrary
|
||||
stride along both row and column, define a mapping function whose storage rank is 3. This permits
|
||||
two elements of the stride vector to have a non-unit value. The map function defined in
|
||||
`cutlass::MatrixTraits::ContiguousLayout` is an example.
|
||||
|
||||
```
|
||||
/// Mapping function for scenario in which layout is row-major or column-major but this information
|
||||
/// is only available at runtime.
|
||||
struct ContiguousLayout {
|
||||
/// Arbitrary storage rank
|
||||
static int const kStorageRank = 3;
|
||||
|
||||
/// Dimension of rows
|
||||
static int const kRow = 0;
|
||||
|
||||
/// Dimension of columns
|
||||
static int const kColumn = 1;
|
||||
|
||||
/// Mapping function defined by runtime variable. Returns coordinates in n-D storage array
|
||||
/// as (matrix row, matrix colum, 0)
|
||||
CUTLASS_HOST_DEVICE
|
||||
Coord<kStorageRank> operator()(MatrixCoord const &coord) const {
|
||||
return make_Coord(coord.row(), coord.column(), 0);
|
||||
}
|
||||
|
||||
/// Helper to construct a stride vector based on contiguous matrix layout and leading dimension
|
||||
CUTLASS_HOST_DEVICE
|
||||
static Coord<kStorageRank> stride(MatrixLayout::Kind layout, int ldm) {
|
||||
if (layout == MatrixLayout::kRowMajor) {
|
||||
return make_Coord(ldm, 1, 1);
|
||||
}
|
||||
return make_Coord(1, ldm, 1);
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
cutlass::TensorView<> extends this concept by including a size vector to specify the bounds of
|
||||
the index space. The value of each coordinate in the size vector defines the half-open range of
|
||||
indices whose smallest value is zero.
|
||||
*/
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// Standard Library includes
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
|
||||
//
|
||||
// CUTLASS includes
|
||||
//
|
||||
|
||||
// Defines cutlass::Coord<>
|
||||
#include "cutlass/coord.h"
|
||||
|
||||
// Defines cutlass::TensorRef<>
|
||||
#include "cutlass/tensor_ref.h"
|
||||
|
||||
// Defines cutlass::TensorView<>
|
||||
#include "cutlass/tensor_view.h"
|
||||
|
||||
// Defines cutlass::MatrixLayout
|
||||
#include "cutlass/matrix_traits.h"
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Column-major matrix access
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Define a rank=2 tensor modeling a column-major matrix
|
||||
typedef cutlass::TensorView<
|
||||
int, // storage element is of type int
|
||||
2, // tensor has rank=2 logical index space
|
||||
cutlass::MatrixLayout::ColumnMajor // column-major mapping function
|
||||
> TensorViewColumnMajor;
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Kernel to copy a matrix from raw memory into a cutlass::TensorView
|
||||
__global__ void MatrixCopyColumnMajor(
|
||||
TensorViewColumnMajor destination, // destination tensor accessed by TensorView
|
||||
int const *source, // source matrix accessed using cuBLAS-style pointer
|
||||
int ldm) { // and leading dimension
|
||||
|
||||
// Compute unique row and column for each thread
|
||||
int row = threadIdx.x + blockIdx.x * blockDim.x;
|
||||
int column = threadIdx.y + blockIdx.y * blockDim.y;
|
||||
|
||||
// Define a coordinate based on the thread's row and column
|
||||
cutlass::Coord<2> coord = cutlass::make_Coord(row, column);
|
||||
|
||||
// Bounds test
|
||||
if (coord < destination.size()) {
|
||||
|
||||
// Access the element
|
||||
destination.at(coord) = source[row + column * ldm];
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Launches kernel MatrixCopyColumnMajor()
|
||||
cudaError_t TestMatrixCopyColumnMajor() {
|
||||
cudaError_t result;
|
||||
|
||||
int const M = 32; // number of rows
|
||||
int const N = 16; // number of columns
|
||||
|
||||
int const ldm = 40; // matrix leading dimension
|
||||
|
||||
//
|
||||
// Allocate source and destination matrices
|
||||
//
|
||||
|
||||
int *Destination;
|
||||
int *Source;
|
||||
|
||||
int const matrix_capacity = ldm * N; // number of elements in memory needed to store matrix
|
||||
size_t const sizeof_matrix = sizeof(int) * matrix_capacity; // size of matrix in bytes
|
||||
|
||||
// Allocate destination and source matrices
|
||||
result = cudaMalloc((void **)&Destination, sizeof_matrix);
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Failed to allocate destination matrix on device: " << cudaGetErrorString(result) << std::endl;
|
||||
return result;
|
||||
}
|
||||
|
||||
result = cudaMalloc((void **)&Source, sizeof_matrix);
|
||||
if (result != cudaSuccess) {
|
||||
cudaFree(Destination);
|
||||
std::cerr << "Failed to allocate source matrix on device:" << cudaGetErrorString(result) << std::endl;
|
||||
return result;
|
||||
}
|
||||
|
||||
// Clear destination matrix in device memory
|
||||
result = cudaMemset(Destination, 0, sizeof_matrix);
|
||||
if (result != cudaSuccess) {
|
||||
cudaFree(Destination);
|
||||
cudaFree(Source);
|
||||
std::cerr << "Failed to clear destination matrix: " << cudaGetErrorString(result) << std::endl;
|
||||
return result;
|
||||
}
|
||||
|
||||
//
|
||||
// Initialize matrix
|
||||
//
|
||||
|
||||
std::vector<int> source_host(matrix_capacity, 0);
|
||||
|
||||
// Procedurally generate input results using several arbitrary constants.
|
||||
int const magic_row_stride = 2;
|
||||
int const magic_column_stride = 3;
|
||||
|
||||
for (int j = 0; j < N; ++j) {
|
||||
for (int i = 0; i < M; ++i) {
|
||||
source_host.at(i + j * ldm) = i * magic_row_stride + j * magic_column_stride;
|
||||
}
|
||||
}
|
||||
|
||||
// Copy to device memory
|
||||
result = cudaMemcpy(Source, source_host.data(), sizeof_matrix, cudaMemcpyHostToDevice);
|
||||
if (result != cudaSuccess) {
|
||||
cudaFree(Destination);
|
||||
cudaFree(Source);
|
||||
std::cerr << "Failed to copy from host to source matrix: " << cudaGetErrorString(result) << std::endl;
|
||||
return result;
|
||||
}
|
||||
|
||||
//
|
||||
// Define a TensorView<> pointing to the destination matrix
|
||||
//
|
||||
TensorViewColumnMajor destination_view_device(
|
||||
Destination, // pointer to base of matrix in device memory
|
||||
cutlass::make_Coord(ldm, 1), // stride vector
|
||||
cutlass::make_Coord(M, N) // bounds of matrix
|
||||
);
|
||||
|
||||
//
|
||||
// Launch kernel to copy matrix
|
||||
//
|
||||
|
||||
dim3 block(16, 16);
|
||||
dim3 grid((M + block.x - 1) / block.x, (N + block.y - 1) / block.y);
|
||||
|
||||
MatrixCopyColumnMajor<<< grid, block >>>(destination_view_device, Source, ldm);
|
||||
|
||||
result = cudaGetLastError();
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Kernel MatrixCopyColumnMajor() failed: "
|
||||
<< cudaGetErrorString(result) << std::endl;
|
||||
|
||||
cudaFree(Destination);
|
||||
cudaFree(Source);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
//
|
||||
// Copy results to host memory
|
||||
//
|
||||
|
||||
std::vector<int> dest_host(matrix_capacity, 0);
|
||||
|
||||
result = cudaMemcpy(dest_host.data(), Destination, sizeof_matrix, cudaMemcpyDeviceToHost);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Failed to copy destination matrix to host memory: "
|
||||
<< cudaGetErrorString(result) << std::endl;
|
||||
|
||||
cudaFree(Destination);
|
||||
cudaFree(Source);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
//
|
||||
// Verify result
|
||||
//
|
||||
|
||||
// Define a TensorView for use in accessing host memory
|
||||
TensorViewColumnMajor destination_view_host(
|
||||
dest_host.data(), // pointer to base of matrix in host memory
|
||||
cutlass::make_Coord(ldm, 1), // stride vector
|
||||
cutlass::make_Coord(M, N) // bounds of matrix
|
||||
);
|
||||
|
||||
// Verify against procedurally computed results
|
||||
for (int j = 0; j < N; ++j) {
|
||||
for (int i = 0; i < M; ++i) {
|
||||
|
||||
// computed result
|
||||
int expected = i * magic_row_stride + j * magic_column_stride;
|
||||
|
||||
// access data by computing explicit offsets
|
||||
int got_explicit = dest_host.at(i + j * ldm);
|
||||
|
||||
// access data in host memory through a TensorView
|
||||
int got_view = destination_view_host.at(cutlass::make_Coord(i, j));
|
||||
|
||||
if (got_explicit != expected) {
|
||||
|
||||
std::cerr << "Error at element (" << i << ", " << j
|
||||
<< ") accessed through explicitly computed offset - expected: " << expected
|
||||
<< ", got: " << got_explicit << std::endl;
|
||||
|
||||
return cudaErrorUnknown;
|
||||
}
|
||||
|
||||
if (got_view != expected) {
|
||||
|
||||
std::cerr << "Error at element (" << i << ", " << j
|
||||
<< ") accesed through TensorView<> on the host - expected: " << expected
|
||||
<< ", got: " << got_view << std::endl;
|
||||
|
||||
return cudaErrorUnknown;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return cudaSuccess;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Entry point for tensor_view example.
|
||||
//
|
||||
// usage:
|
||||
//
|
||||
// 02_tensor_view
|
||||
//
|
||||
int main() {
|
||||
|
||||
cudaError_t result = TestMatrixCopyColumnMajor();
|
||||
|
||||
if (result == cudaSuccess) {
|
||||
std::cout << "Passed" << std::endl;
|
||||
}
|
||||
|
||||
return (result == cudaSuccess ? 0 : -1);
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,38 @@
|
||||
# Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
# provided that the following conditions are met:
|
||||
# * Redistributions of source code must retain the above copyright notice, this list of
|
||||
# conditions and the following disclaimer.
|
||||
# * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
# conditions and the following disclaimer in the documentation and/or other materials
|
||||
# provided with the distribution.
|
||||
# * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
# to endorse or promote products derived from this software without specific prior written
|
||||
# permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
set(EXAMPLES_CUTLASS_UTILITIES_SOURCES
|
||||
cutlass_utilities.cu
|
||||
)
|
||||
|
||||
if (NOT CUTLASS_NATIVE_CUDA)
|
||||
# cuda_add_executable does not take interface include directories into account
|
||||
# Let's fetch them and pass them to CUDA.
|
||||
get_target_property(CUTLASS_INCLUDES CUTLASS INTERFACE_INCLUDE_DIRECTORIES)
|
||||
include_directories("${CUTLASS_INCLUDES}")
|
||||
endif()
|
||||
|
||||
cutlass_add_executable(
|
||||
02_cutlass_utilities
|
||||
${EXAMPLES_CUTLASS_UTILITIES_SOURCES}
|
||||
)
|
||||
@@ -0,0 +1,359 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*
|
||||
This example demonstrates several CUTLASS utilities in the context of a mixed-precision
|
||||
floating-point matrix product computation.
|
||||
|
||||
These utilities are intended to be useful supporting components for managing tensor and matrix
|
||||
memory allocations, initializing and comparing results, and computing reference output.
|
||||
|
||||
CUTLASS utilities are defined in the directory `tools/util`, and definitions appear
|
||||
namespace `cutlass::` or an inner namespace therein. Operations in `cutlass::reference::` have
|
||||
both host-side and device-side implementations, and the choice to use device-side initialization
|
||||
and host-side verification in this example was arbitrary.
|
||||
|
||||
|
||||
cutlass::half_t
|
||||
|
||||
This is a host-only implementation of a half-precision floating-point type. It requires no
|
||||
specialized hardware support from the CPU and emulates arithmetic operations. Device-side code
|
||||
should use CUDA's `half` type.
|
||||
|
||||
|
||||
cutlass::HostMatrix<>
|
||||
|
||||
This template class simplifies the creation of a rank=2 tensor with either a column-major or
|
||||
row-major layout in memory.
|
||||
|
||||
This class offers methods device_view() and host_view() to provide TensorView objects for
|
||||
device- and host-side memory allocations.
|
||||
|
||||
|
||||
cutlass::reference::device::TensorInitialize()
|
||||
|
||||
This template function initializes the elements of a tensor according to either a procedural
|
||||
definition or a random distribution. The function in namespace `cutlass::reference::device::`
|
||||
uses a CUDA kernel to perform this initialization, relying on CURAND to compute random numbers.
|
||||
|
||||
|
||||
cutlass::reference::host::Gemm()
|
||||
|
||||
This template function computes the general matrix product. This template supports unique
|
||||
data types for each matrix operand, the internal accumulation type, and the scalar parameters
|
||||
alpha and beta.
|
||||
|
||||
|
||||
cutlass::reference::host::TensorEquals()
|
||||
|
||||
Compares two tensors of identical rank and returns true if values are bit equivalent.
|
||||
|
||||
*/
|
||||
|
||||
// Standard Library includes
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <vector>
|
||||
|
||||
// CUTLASS includes needed for mixed-precision GEMM kernel
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/fp16_sgemm_traits.h"
|
||||
|
||||
//
|
||||
// CUTLASS utility includes
|
||||
//
|
||||
|
||||
// Defines operator<<() to write TensorView objects to std::ostream
|
||||
#include "tools/util/tensor_view_io.h"
|
||||
|
||||
// Defines cutlass::HostMatrix<>
|
||||
#include "tools/util/host_matrix.h"
|
||||
|
||||
// Defines cutlass::half_t
|
||||
#include "tools/util/half.h"
|
||||
|
||||
// Defines cutlass::reference::device::TensorInitialize()
|
||||
#include "tools/util/reference/device/tensor_elementwise.h"
|
||||
|
||||
// Defines cutlass::reference::host::TensorEquals()
|
||||
#include "tools/util/reference/host/tensor_elementwise.h"
|
||||
|
||||
// Defines cutlass::reference::host::Gemm()
|
||||
#include "tools/util/reference/host/gemm.h"
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Define a CUTLASS GEMM template and launch a GEMM kernel.
|
||||
cudaError_t Cutlass_FP16_SgemmNN(
|
||||
int M,
|
||||
int N,
|
||||
int K,
|
||||
cutlass::half_t alpha,
|
||||
half const *A,
|
||||
int lda,
|
||||
half const *B,
|
||||
int ldb,
|
||||
cutlass::half_t beta,
|
||||
half *C,
|
||||
int ldc) {
|
||||
|
||||
// Define a CUTLASS Gemm using mixed-precision floating-point.
|
||||
//
|
||||
// A, B, C, D are half-precision. Internal accumulation is in single-precision.
|
||||
//
|
||||
// Note, we use CUDA's `half` type for device-side code including CUTLASS GEMM kernels.
|
||||
//
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, // A type
|
||||
half, // B type
|
||||
half, // C type
|
||||
half, // D type
|
||||
half // Scalar type: alpha, beta
|
||||
>
|
||||
GemmTraits;
|
||||
|
||||
// Define a CUTLASS GEMM object.
|
||||
typedef cutlass::gemm::Gemm<GemmTraits> Gemm;
|
||||
|
||||
// Construct and initialize CUTLASS GEMM parameters object.
|
||||
typename Gemm::Params params;
|
||||
|
||||
int result = params.initialize(
|
||||
M, // GEMM M dimension
|
||||
N, // GEMM N dimension
|
||||
K, // GEMM K dimension
|
||||
half(float(alpha)), // scalar alpha - This is a legal conversion from cutlass::half_t to CUDA's half.
|
||||
A, // matrix A operand
|
||||
lda,
|
||||
B, // matrix B operand
|
||||
ldb,
|
||||
half(float(beta)), // scalar beta - This is a legal conversion from cutlass::half_t to CUDA's half.
|
||||
C, // source matrix C
|
||||
ldc,
|
||||
C, // destination matrix C (may be different memory than source C matrix)
|
||||
ldc
|
||||
);
|
||||
|
||||
if (result) {
|
||||
std::cerr << "Failed to initialize CUTLASS Gemm::Params object." << std::endl;
|
||||
return cudaErrorInvalidValue;
|
||||
}
|
||||
|
||||
// Launch the CUTLASS GEMM kernel.
|
||||
Gemm::launch(params);
|
||||
|
||||
// Return any errors associated with the launch or cudaSuccess if no error.
|
||||
return cudaGetLastError();
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Allocate several matrices in GPU device memory and call a single-precision
|
||||
/// CUTLASS GEMM kernel.
|
||||
cudaError_t TestCutlassGemm(int M, int N, int K, cutlass::half_t alpha, cutlass::half_t beta) {
|
||||
cudaError_t result;
|
||||
|
||||
//
|
||||
// Construct cutlass::HostMatrix<> using the half-precision host-side type.
|
||||
//
|
||||
// cutlass::HostMatrix<> allocates memory on both the host and device corresponding to rank=2
|
||||
// tensors in column-major layout. Explicit synchronization methods are offered to copy the
|
||||
// tensor to the device or to the host.
|
||||
//
|
||||
|
||||
// M-by-K matrix of cutlass::half_t
|
||||
cutlass::HostMatrix<cutlass::half_t> A(cutlass::MatrixCoord(M, K));
|
||||
|
||||
// K-by-N matrix of cutlass::half_t
|
||||
cutlass::HostMatrix<cutlass::half_t> B(cutlass::MatrixCoord(K, N));
|
||||
|
||||
// M-by-N matrix of cutlass::half_t
|
||||
cutlass::HostMatrix<cutlass::half_t> C_cutlass(cutlass::MatrixCoord(M, N));
|
||||
|
||||
// M-by-N matrix of cutlass::half_t
|
||||
cutlass::HostMatrix<cutlass::half_t> C_reference(cutlass::MatrixCoord(M, N));
|
||||
|
||||
//
|
||||
// Initialize matrices with small, random integers.
|
||||
//
|
||||
|
||||
cutlass::Distribution dist;
|
||||
|
||||
// Uniform random distribution from -4 .. 4. Values are truncated to integers.
|
||||
dist.set_uniform(-4, 4);
|
||||
|
||||
// Arbitrary RNG seed value. Hard-coded for deterministic results.
|
||||
int seed = 2080;
|
||||
|
||||
cutlass::reference::device::TensorInitialize(
|
||||
A.device_view(), // concept: TensorView
|
||||
seed,
|
||||
dist);
|
||||
|
||||
cutlass::reference::device::TensorInitialize(
|
||||
B.device_view(), // concept: TensorView
|
||||
seed * 2,
|
||||
dist);
|
||||
cutlass::reference::device::TensorInitialize(
|
||||
C_cutlass.device_view(), // concept: TensorView
|
||||
seed * 3,
|
||||
dist);
|
||||
|
||||
// Copy C_cutlass into C_reference so the GEMM is correct when beta != 0.
|
||||
cutlass::reference::device::TensorFill(C_reference.device_view(), C_cutlass.device_view());
|
||||
|
||||
// Copy the device-side view into host memory
|
||||
C_reference.sync_host();
|
||||
|
||||
//
|
||||
// Launch the CUTLASS GEMM kernel
|
||||
//
|
||||
|
||||
result = Cutlass_FP16_SgemmNN(
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
alpha,
|
||||
A.device_data(),
|
||||
A.leading_dim(),
|
||||
B.device_data(),
|
||||
B.leading_dim(),
|
||||
beta,
|
||||
C_cutlass.device_data(),
|
||||
C_cutlass.leading_dim()
|
||||
);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
return result;
|
||||
}
|
||||
|
||||
//
|
||||
// Verify the result using a host-side reference
|
||||
//
|
||||
|
||||
// A and B were initialized using device-side procedures. The intent of this example is to
|
||||
// use the host-side reference GEMM, so we must perform a device-to-host copy.
|
||||
A.sync_host();
|
||||
B.sync_host();
|
||||
|
||||
// Copy CUTLASS's GEMM results into host memory.
|
||||
C_cutlass.sync_host();
|
||||
|
||||
// Compute the reference result using the host-side GEMM reference implementation.
|
||||
cutlass::reference::host::Gemm(
|
||||
cutlass::gemm::GemmCoord(K, N, M), // problem size (type: cutlass::gemm::GemmCoord)
|
||||
alpha, // alpha (type: cutlass::half_t)
|
||||
A.host_ref(), // A (concept: TensorRef)
|
||||
B.host_ref(), // B (concept: TensorRef)
|
||||
beta, // beta (type: cutlass::half_t)
|
||||
C_reference.host_ref(), // C (concept: TensorRef)
|
||||
float(0) // Accumulator initial value passed as argument to deduce
|
||||
); // internal accumulation data type as float.
|
||||
|
||||
// Compare reference to computed results.
|
||||
if (!cutlass::reference::host::TensorEquals(C_reference.host_view(), C_cutlass.host_view())) {
|
||||
|
||||
std::cerr << "Error - CUTLASS mixed-precision GEMM kernel differs from reference." << std::endl;
|
||||
|
||||
//
|
||||
// On error, print C_cutlass and C_reference to std::cerr.
|
||||
//
|
||||
// Note, these are matrices of half-precision elements stored in host memory as
|
||||
// arrays of type cutlass::half_t.
|
||||
//
|
||||
|
||||
// Result of CUTLASS mixed-precision GEMM kernel
|
||||
std::cerr << "CUTLASS:\n" << C_cutlass << std::endl;
|
||||
|
||||
// Result of reference computation
|
||||
std::cerr << "Reference:\n" << C_reference << std::endl;
|
||||
|
||||
// Return error code.
|
||||
return cudaErrorUnknown;
|
||||
}
|
||||
|
||||
// Passed error check
|
||||
return cudaSuccess;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Entry point to cutlass_utilities example.
|
||||
//
|
||||
// usage:
|
||||
//
|
||||
// 01_cutlass_utilities <M> <N> <K> <alpha> <beta>
|
||||
//
|
||||
int main(int argc, const char *arg[]) {
|
||||
|
||||
//
|
||||
// Parse the command line to obtain GEMM dimensions and scalar values.
|
||||
//
|
||||
|
||||
// GEMM problem dimensions: <M> <N> <K>
|
||||
int problem[3] = { 128, 128, 128 };
|
||||
|
||||
for (int i = 1; i < argc && i < 4; ++i) {
|
||||
std::stringstream ss(arg[i]);
|
||||
ss >> problem[i - 1];
|
||||
}
|
||||
|
||||
// Linear scale factors in GEMM. Note, these are half-precision values stored as
|
||||
// cutlass::half_t.
|
||||
//
|
||||
// Values outside the range of IEEE FP16 will overflow to infinity or underflow to zero.
|
||||
//
|
||||
cutlass::half_t scalars[2] = { 1, 0 };
|
||||
|
||||
for (int i = 4; i < argc && i < 6; ++i) {
|
||||
std::stringstream ss(arg[i]);
|
||||
|
||||
ss >> scalars[i - 4]; // lexical cast to cutlass::half_t
|
||||
}
|
||||
|
||||
//
|
||||
// Run the CUTLASS GEMM test.
|
||||
//
|
||||
|
||||
cudaError_t result = TestCutlassGemm(
|
||||
problem[0], // GEMM M dimension
|
||||
problem[1], // GEMM N dimension
|
||||
problem[2], // GEMM K dimension
|
||||
scalars[0], // alpha
|
||||
scalars[1] // beta
|
||||
);
|
||||
|
||||
if (result == cudaSuccess) {
|
||||
std::cout << "Passed." << std::endl;
|
||||
}
|
||||
|
||||
// Exit.
|
||||
return result == cudaSuccess ? 0 : -1;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,38 @@
|
||||
# Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
# provided that the following conditions are met:
|
||||
# * Redistributions of source code must retain the above copyright notice, this list of
|
||||
# conditions and the following disclaimer.
|
||||
# * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
# conditions and the following disclaimer in the documentation and/or other materials
|
||||
# provided with the distribution.
|
||||
# * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
# to endorse or promote products derived from this software without specific prior written
|
||||
# permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
set(EXAMPLES_STRIDED_BATCHED_GEMM_SOURCES
|
||||
strided_batched_gemm.cu
|
||||
)
|
||||
|
||||
if (NOT CUTLASS_NATIVE_CUDA)
|
||||
# cuda_add_executable does not take interface include directories into account
|
||||
# Let's fetch them and pass them to CUDA.
|
||||
get_target_property(CUTLASS_INCLUDES CUTLASS INTERFACE_INCLUDE_DIRECTORIES)
|
||||
include_directories("${CUTLASS_INCLUDES}")
|
||||
endif()
|
||||
|
||||
cutlass_add_executable(
|
||||
03_strided_batched_gemm
|
||||
${EXAMPLES_STRIDED_BATCHED_GEMM_SOURCES}
|
||||
)
|
||||
@@ -0,0 +1,349 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/sgemm_traits.h"
|
||||
|
||||
/*
|
||||
This example demonstrates how to use cutlass to compute a batched strided gemm.
|
||||
In this example, both A and B matrix are non-transpose and column major matrix
|
||||
batched_C = batched_A x batched_B
|
||||
As an example, matrix C can be seen as
|
||||
-----------------------------------------------------------
|
||||
(0,0,0) | (0,0,1) | (0,0,2) | (1,0,0) | (1,0,1) | (1,0,2) |
|
||||
-----------------------------------------------------------
|
||||
(0,1,0) | (0,1,1) | (0,1,2) | (1,1,0) | (1,1,1) | (1,1,2) |
|
||||
-----------------------------------------------------------
|
||||
(0,2,0) | (0,2,1) | (0,2,2) | (1,2,0) | (1,2,1) | (1,2,2) |
|
||||
-----------------------------------------------------------
|
||||
(0,3,0) | (0,3,1) | (0,3,2) | (1,3,0) | (1,3,1) | (1,3,2) |
|
||||
-----------------------------------------------------------
|
||||
(0,4,0) | (0,4,1) | (0,4,2) | (1,4,0) | (1,4,1) | (1,4,2) |
|
||||
-----------------------------------------------------------
|
||||
(0,5,0) | (0,5,1) | (0,5,2) | (1,5,0) | (1,5,1) | (1,5,2) |
|
||||
-----------------------------------------------------------
|
||||
batch 0 | batch 1
|
||||
where we denote each element with (batch_idx, row_idx, column_idx)
|
||||
In this example, batch size is 2, M is 6 and N is 3
|
||||
The stride (batch_stride_C) between the first element of two batches is ldc * n
|
||||
|
||||
matrix A can be seen as
|
||||
---------------------------------------
|
||||
(0,0,0) | (0,0,1) | (1,0,0) | (1,0,1) |
|
||||
---------------------------------------
|
||||
(0,1,0) | (0,1,1) | (1,1,0) | (1,1,1) |
|
||||
---------------------------------------
|
||||
(0,2,0) | (0,2,1) | (1,2,0) | (1,2,1) |
|
||||
---------------------------------------
|
||||
(0,3,0) | (0,3,1) | (1,3,0) | (1,3,1) |
|
||||
---------------------------------------
|
||||
(0,4,0) | (0,4,1) | (1,4,0) | (1,4,1) |
|
||||
---------------------------------------
|
||||
(0,5,0) | (0,5,1) | (1,5,0) | (1,5,1) |
|
||||
---------------------------------------
|
||||
batch 0 | batch 1
|
||||
, where batch size is 2, M is 6 and K is 2
|
||||
The stride (batch_stride_B) between the first element of two batches is lda * k
|
||||
|
||||
matrix B can be seen as
|
||||
-----------------------------
|
||||
(0,0,0) | (0,0,1) | (0,0,2) |
|
||||
----------------------------- batch 0
|
||||
(0,1,0) | (0,1,1) | (0,1,2) |
|
||||
-------------------------------------
|
||||
(1,0,0) | (1,0,1) | (1,0,2) |
|
||||
----------------------------- batch 1
|
||||
(1,1,0) | (1,1,1) | (1,1,2) |
|
||||
-----------------------------
|
||||
, where the batch size is 2, N is 3 and K is 2
|
||||
The stride (batch_stride_C) between the first element of two batches is k
|
||||
|
||||
|
||||
*/
|
||||
|
||||
cudaError_t cutlass_strided_batched_sgemm(float const *A,
|
||||
int lda,
|
||||
long long int batch_stride_A,
|
||||
float const *B,
|
||||
int ldb,
|
||||
long long int batch_stride_B,
|
||||
float *C,
|
||||
int ldc,
|
||||
long long int batch_stride_C,
|
||||
float alpha,
|
||||
float beta,
|
||||
int m,
|
||||
int n,
|
||||
int k,
|
||||
int batch_count) {
|
||||
// create a cutlass traits
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
|
||||
SgemmTraits;
|
||||
|
||||
// create a CUTLASS GEMM object.
|
||||
typedef cutlass::gemm::Gemm<SgemmTraits> Gemm;
|
||||
|
||||
// Construct and initialize CUTLASS GEMM parameters object.
|
||||
typename Gemm::Params params;
|
||||
|
||||
int result = params.initialize(
|
||||
m, // M dimension for each batch
|
||||
n, // N dimension for each batch
|
||||
k, // K dimension for each batch
|
||||
alpha, // scalar alpha
|
||||
A,
|
||||
lda,
|
||||
batch_stride_A, // distance in memory between the first element of neighboring batch
|
||||
B,
|
||||
ldb,
|
||||
batch_stride_B, // distance in memory between the first element of neighboring batch
|
||||
beta, // scalar beta
|
||||
C, // source matrix C
|
||||
ldc,
|
||||
batch_stride_C, // distance in memory between the first element of neighboring batch
|
||||
C, // destination matrix C (may be different memory than source C matrix)
|
||||
ldc,
|
||||
batch_stride_C, // distance in memory between the first element of neighboring batch
|
||||
batch_count
|
||||
);
|
||||
|
||||
if (result != 0) {
|
||||
std::cerr << "Failed to initialize CUTLASS Gemm::Params object." << std::endl;
|
||||
return cudaErrorInvalidValue;
|
||||
}
|
||||
|
||||
// Launch the CUTLASS GEMM kernel.
|
||||
Gemm::launch(params);
|
||||
result = cudaDeviceSynchronize();
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "kernel launch result = " << result << std::endl;
|
||||
}
|
||||
return cudaGetLastError();
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
cudaError_t strided_batched_gemm_nn_reference(std::vector<T> const &A,
|
||||
int lda,
|
||||
long long int batch_stride_A,
|
||||
std::vector<T> const &B,
|
||||
int ldb,
|
||||
long long int batch_stride_B,
|
||||
std::vector<T> &C,
|
||||
int ldc,
|
||||
long long int batch_stride_C,
|
||||
T alpha,
|
||||
T beta,
|
||||
int m,
|
||||
int n,
|
||||
int k,
|
||||
int batch_count) {
|
||||
/*
|
||||
strided batched gemm NN
|
||||
*/
|
||||
|
||||
cudaError_t result = cudaSuccess;
|
||||
|
||||
if (A.size() < lda * k * batch_count) {
|
||||
std::cout << "the size of A is too small" << std::endl;
|
||||
return cudaErrorInvalidValue;
|
||||
}
|
||||
if (B.size() < ldb * n) {
|
||||
std::cout << "the size of B is too small" << std::endl;
|
||||
return cudaErrorInvalidValue;
|
||||
}
|
||||
if (C.size() < ldc * n * batch_count) {
|
||||
std::cout << "the size of C is too small" << std::endl;
|
||||
return cudaErrorInvalidValue;
|
||||
}
|
||||
|
||||
for (int batch_idx = 0; batch_idx < batch_count; batch_idx++) {
|
||||
for (int n_idx = 0; n_idx < n; n_idx++) {
|
||||
for (int m_idx = 0; m_idx < m; m_idx++) {
|
||||
T accum = beta * C[batch_idx * batch_stride_C + n_idx * ldc + m_idx];
|
||||
for (int k_idx = 0; k_idx < k; k_idx++) {
|
||||
accum += alpha
|
||||
* A[batch_idx * batch_stride_A + k_idx * lda + m_idx]
|
||||
* B[batch_idx * batch_stride_B + n_idx * ldb + k_idx];
|
||||
}
|
||||
C[batch_idx * batch_stride_C + n_idx * ldc + m_idx] = accum;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
int main() {
|
||||
int const m = 16;
|
||||
int const n = 24;
|
||||
int const k = 8;
|
||||
int const batch_count = 3;
|
||||
|
||||
// A, B are non-transpose, column major
|
||||
int const lda = m;
|
||||
int const ldb = k * batch_count;
|
||||
int const ldc = m;
|
||||
|
||||
int const count_A = batch_count * lda * k;
|
||||
int const count_B = ldb * n;
|
||||
int const count_C = batch_count * ldc * n;
|
||||
|
||||
// the memory is batched along K dimension
|
||||
long long int batch_stride_A = static_cast<long long int>(lda) * static_cast<long long int>(k);
|
||||
long long int batch_stride_B = static_cast<long long int>(k);
|
||||
long long int batch_stride_C = static_cast<long long int>(ldc) * static_cast<long long int>(n);
|
||||
|
||||
// alpha and beta
|
||||
float alpha = 1.0f;
|
||||
float beta = 2.0f;
|
||||
|
||||
cudaError_t result = cudaSuccess;
|
||||
|
||||
// allocate the host memory
|
||||
std::vector<float> host_A(count_A);
|
||||
std::vector<float> host_B(count_B);
|
||||
std::vector<float> host_C(count_C);
|
||||
std::vector<float> result_C(count_C);
|
||||
|
||||
// allocate the device memory
|
||||
float *A;
|
||||
float *B;
|
||||
float *C;
|
||||
|
||||
result = cudaMalloc(&A, count_A * sizeof(float));
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "cudaMalloc result = " << result << std::endl;
|
||||
return result;
|
||||
}
|
||||
result = cudaMalloc(&B, count_B * sizeof(float));
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "cudaMalloc result = " << result << std::endl;
|
||||
return result;
|
||||
}
|
||||
result = cudaMalloc(&C, count_C * sizeof(float));
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "cudaMalloc result = " << result << std::endl;
|
||||
return result;
|
||||
}
|
||||
|
||||
// fill A
|
||||
for (int b_idx = 0; b_idx < batch_count; b_idx++) {
|
||||
for (int col_idx = 0; col_idx < k; col_idx++) {
|
||||
for (int row_idx = 0; row_idx < m; row_idx++) {
|
||||
host_A[row_idx + col_idx * lda + b_idx * lda * k] = static_cast<float>(row_idx + col_idx * lda + b_idx * lda * k);
|
||||
}
|
||||
}
|
||||
}
|
||||
// fill B
|
||||
for (int b_idx = 0; b_idx < batch_count; b_idx++) {
|
||||
for (int col_idx = 0; col_idx < n; col_idx++) {
|
||||
for (int row_idx = 0; row_idx < k; row_idx++) {
|
||||
host_B[row_idx + col_idx * ldb + b_idx * k] = static_cast<float>(n + k * ldb + batch_count * k) - static_cast<float>(row_idx + col_idx * ldb + b_idx * k);
|
||||
}
|
||||
}
|
||||
}
|
||||
// fill C
|
||||
for (int b_idx = 0; b_idx < batch_count; b_idx++) {
|
||||
for (int col_idx = 0; col_idx < n; col_idx++) {
|
||||
for (int row_idx = 0; row_idx < m; row_idx++) {
|
||||
host_C[row_idx + col_idx * ldc + b_idx * ldc * n] = 1.f;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ref memory
|
||||
std::vector<float> ref_A(host_A);
|
||||
std::vector<float> ref_B(host_B);
|
||||
std::vector<float> ref_C(host_C);
|
||||
// copy host memory to device
|
||||
result = cudaMemcpy(A, host_A.data(), count_A * sizeof(float), cudaMemcpyHostToDevice);
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "cudaMemcpy result = " << result << std::endl;
|
||||
return result;
|
||||
}
|
||||
result = cudaMemcpy(B, host_B.data(), count_B * sizeof(float), cudaMemcpyHostToDevice);
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "cudaMemcpy result = " << result << std::endl;
|
||||
return result;
|
||||
}
|
||||
result = cudaMemcpy(C, host_C.data(), count_C * sizeof(float), cudaMemcpyHostToDevice);
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "cudaMemcpy result = " << result << std::endl;
|
||||
return result;
|
||||
}
|
||||
|
||||
// run cutlass
|
||||
result = cutlass_strided_batched_sgemm(A, lda, batch_stride_A, B, ldb, batch_stride_B, C, ldc, batch_stride_C,
|
||||
alpha, beta, m, n, k, batch_count);
|
||||
if (result != cudaSuccess)
|
||||
return result;
|
||||
|
||||
// copy device memory to host
|
||||
result = cudaMemcpy(result_C.data(), C, count_C * sizeof(float), cudaMemcpyDeviceToHost);
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "cudaMemcpy result = " << result << std::endl;
|
||||
return result;
|
||||
}
|
||||
|
||||
//compare with reference code
|
||||
result = strided_batched_gemm_nn_reference(ref_A, lda, batch_stride_A, ref_B, ldb, batch_stride_B, ref_C, ldc, batch_stride_C,
|
||||
alpha, beta, m, n, k, batch_count);
|
||||
if (result != 0)
|
||||
return result;
|
||||
|
||||
if (ref_C != result_C) {
|
||||
std::cout << "CUTLASS strided batched gemm does not run correctly" << std::endl;
|
||||
return cudaErrorUnknown;
|
||||
}
|
||||
|
||||
// free memory
|
||||
result = cudaFree(A);
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "cudaFree result = " << result << std::endl;
|
||||
return result;
|
||||
}
|
||||
result = cudaFree(B);
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "cudaFree result = " << result << std::endl;
|
||||
return result;
|
||||
}
|
||||
result = cudaFree(C);
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "cudaFree result = " << result << std::endl;
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
if (result == cudaSuccess) {
|
||||
std::cout << "Passed." << std::endl;
|
||||
}
|
||||
|
||||
// Exit.
|
||||
return result == cudaSuccess ? 0 : -1;
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
# Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
# provided that the following conditions are met:
|
||||
# * Redistributions of source code must retain the above copyright notice, this list of
|
||||
# conditions and the following disclaimer.
|
||||
# * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
# conditions and the following disclaimer in the documentation and/or other materials
|
||||
# provided with the distribution.
|
||||
# * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
# to endorse or promote products derived from this software without specific prior written
|
||||
# permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
set(EXAMPLES_BASIC_CUTLASS_GEMM_SOURCES
|
||||
tile_iterator.cu
|
||||
)
|
||||
|
||||
if (NOT CUTLASS_NATIVE_CUDA)
|
||||
# cuda_add_executable does not take interface include directories into account
|
||||
# Let's fetch them and pass them to CUDA.
|
||||
get_target_property(CUTLASS_INCLUDES CUTLASS INTERFACE_INCLUDE_DIRECTORIES)
|
||||
include_directories("${CUTLASS_INCLUDES}")
|
||||
endif()
|
||||
|
||||
cutlass_add_executable(
|
||||
04_tile_iterator
|
||||
${EXAMPLES_BASIC_CUTLASS_GEMM_SOURCES}
|
||||
)
|
||||
@@ -0,0 +1,248 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*
|
||||
This example demonstrates how to use the TileIterator in CUTLASS to load data from addressable
|
||||
memory, and store it back into addressable memory.
|
||||
|
||||
TileIterator is a core concept in CUTLASS that enables efficient loading and storing of data from
|
||||
and to addressable memory. The TileIterator accepts a TileTraits type, which defines the shape of a
|
||||
tile and the distribution of accesses by individual entities, either threads or others.
|
||||
|
||||
In this example, a LoadTileIterator is used to load elements from a tile in global memory, stored in
|
||||
column-major layout, into a fragment, and a corresponding StoreTileIterator is used to store the
|
||||
elements back into global memory (in the same column-major layout).
|
||||
|
||||
https://devblogs.nvidia.com/cutlass-linear-algebra-cuda/
|
||||
|
||||
This example uses CUTLASS utilities to ease the matrix operations.
|
||||
*/
|
||||
|
||||
// Standard Library includes
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <vector>
|
||||
|
||||
// CUTLASS includes
|
||||
#include "cutlass/tile_iterator.h"
|
||||
#include "cutlass/tile_traits_standard.h"
|
||||
|
||||
//
|
||||
// CUTLASS utility includes
|
||||
//
|
||||
|
||||
// Defines operator<<() to write TensorView objects to std::ostream
|
||||
#include "tools/util/tensor_view_io.h"
|
||||
|
||||
// Defines cutlass::HostMatrix<>
|
||||
#include "tools/util/host_matrix.h"
|
||||
|
||||
// Defines cutlass::reference::device::TensorInitialize()
|
||||
#include "tools/util/reference/device/tensor_elementwise.h"
|
||||
|
||||
// Defines cutlass::reference::host::TensorEquals()
|
||||
#include "tools/util/reference/host/tensor_elementwise.h"
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// This function defines load and store tile iterators to load and store a M-by-K tile, in
|
||||
// column-major layout, from and back into global memory.
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename Traits>
|
||||
__global__ void cutlass_tile_iterator_load_store_global(
|
||||
float const *input,
|
||||
float *output,
|
||||
int M,
|
||||
int K) {
|
||||
|
||||
// Define a tile load iterator
|
||||
typedef cutlass::TileLoadIterator<
|
||||
Traits, // the Traits type, defines shape/distribution of accesses
|
||||
float, // elements are of type float
|
||||
cutlass::IteratorAdvance::kH, // post-increment accesses advance in strided (as opposed to
|
||||
// contiguous dimension
|
||||
cutlass::MemorySpace::kGlobal // iterator loads from global memory
|
||||
> TileLoadIterator;
|
||||
|
||||
// Defines a tile store iterator
|
||||
typedef cutlass::TileStoreIterator<
|
||||
Traits, // the Traits type, defines shape/distribution of accesses
|
||||
float, // elements are of type float
|
||||
cutlass::IteratorAdvance::kH, // post-increment accesses advance in strided (as opposed to
|
||||
// contiguous) dimension
|
||||
cutlass::MemorySpace::kGlobal // iterator stores into global memory
|
||||
> TileStoreIterator;
|
||||
|
||||
// Defines a predicate vector for managing statically sized vector of boolean predicates
|
||||
typedef typename TileLoadIterator::PredicateVector PredicateVector;
|
||||
|
||||
// The parameters specified to the iterators. These include the pointer to the source of
|
||||
// addressable memory, and the strides and increments for each of the tile's dimensions
|
||||
typename TileLoadIterator::Params load_params;
|
||||
typename TileStoreIterator::Params store_params;
|
||||
|
||||
// Initializing the parameters for both of the iterators. The TileLoadIterator accesses the
|
||||
// input matrix and TileStoreIterator accesses the output matrix. The strides are set
|
||||
// identically since the data is being stored in the same way as it is loaded (column-major
|
||||
// mapping).
|
||||
load_params.initialize(input, M*K, M, 1);
|
||||
store_params.initialize(output, M*K, M, 1);
|
||||
|
||||
// Constructing the tile load and store iterators, and the predicates vector
|
||||
TileLoadIterator load_iterator(load_params);
|
||||
TileStoreIterator store_iterator(store_params);
|
||||
PredicateVector predicates;
|
||||
|
||||
// Initializing the predicates with bounds set to <1, K, M>. This protects out-of-bounds loads.
|
||||
load_iterator.initialize_predicates(predicates.begin(), cutlass::make_Coord(1, K, M));
|
||||
|
||||
// The fragment in which the elements are loaded into and stored from.
|
||||
typename TileLoadIterator::Fragment fragment;
|
||||
|
||||
// Loading a tile into a fragment and advancing to the next tile's position
|
||||
load_iterator.load_post_increment(fragment, predicates.begin());
|
||||
// Storing a tile from fragment and advancing to the next tile's position
|
||||
store_iterator.store_post_increment(fragment);
|
||||
}
|
||||
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// Launches cutlass_tile_iterator_load_store_global kernel
|
||||
cudaError_t test_cutlass_tile_iterator() {
|
||||
cudaError_t result = cudaSuccess;
|
||||
|
||||
// Creating a M-by-K (128-by-8) tile for this example.
|
||||
static int const M = 128;
|
||||
static int const K = 8;
|
||||
// The kernel is launched with 128 threads per thread block.
|
||||
static int const kThreadsPerThreadBlock = 128;
|
||||
// Define the tile type
|
||||
typedef cutlass::Shape<1, 8, 128> Tile;
|
||||
|
||||
// CUTLASS provides a standard TileTraits type, which chooses the 'best' shape to enable warp
|
||||
// raking along the contiguous dimension if possible.
|
||||
typedef cutlass::TileTraitsStandard<Tile, kThreadsPerThreadBlock> Traits;
|
||||
|
||||
// M-by-K input matrix of float
|
||||
cutlass::HostMatrix<float> input(cutlass::MatrixCoord(M, K));
|
||||
|
||||
// M-by-K output matrix of float
|
||||
cutlass::HostMatrix<float> output(cutlass::MatrixCoord(M, K));
|
||||
|
||||
//
|
||||
// Initialize input matrix with linear combination.
|
||||
//
|
||||
|
||||
cutlass::Distribution dist;
|
||||
|
||||
// Linear distribution in column-major format.
|
||||
dist.set_linear(1, 1, M);
|
||||
|
||||
// Arbitrary RNG seed value. Hard-coded for deterministic results.
|
||||
int seed = 2080;
|
||||
|
||||
cutlass::reference::device::TensorInitialize(
|
||||
input.device_view(), // concept: TensorView
|
||||
seed,
|
||||
dist);
|
||||
|
||||
// Initialize output matrix to all zeroes.
|
||||
output.fill(0);
|
||||
|
||||
// Launch kernel to load and store tiles from/to global memory.
|
||||
cutlass_tile_iterator_load_store_global<Traits><<<
|
||||
dim3(1, 1, 1),
|
||||
dim3(kThreadsPerThreadBlock, 1)
|
||||
>>>(input.device_data(), output.device_data(), M, K);
|
||||
|
||||
result = cudaDeviceSynchronize();
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
return result;
|
||||
}
|
||||
|
||||
// Copy results to host
|
||||
output.sync_host();
|
||||
|
||||
// Verify results
|
||||
for(int i = 0; i < M; ++i) {
|
||||
for(int j = 0; j < K; ++j) {
|
||||
if(output.at(cutlass::make_Coord(i, j)) != float(M*j+i+1)){
|
||||
std::cout << "FAILED: (" << i << ", " << j
|
||||
<< ") -- expected: " << (M*j+i+1)
|
||||
<< ", actual: " << output.at(cutlass::make_Coord(i, j))
|
||||
<< std::endl;
|
||||
result = cudaErrorUnknown;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Entry point to tile_iterator example.
|
||||
//
|
||||
// usage:
|
||||
//
|
||||
// 04_tile_iterator
|
||||
//
|
||||
int main(int argc, const char *arg[]) {
|
||||
|
||||
// Properties of CUDA device
|
||||
cudaDeviceProp device_properties;
|
||||
|
||||
// Assumne the device id is 0.
|
||||
int device_id = 0;
|
||||
|
||||
cudaError_t result = cudaGetDeviceProperties(&device_properties, device_id);
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Failed to get device properties: "
|
||||
<< cudaGetErrorString(result) << std::endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// Run the CUTLASS tile iterator test.
|
||||
//
|
||||
|
||||
result = test_cutlass_tile_iterator();
|
||||
|
||||
if (result == cudaSuccess) {
|
||||
std::cout << "Passed." << std::endl;
|
||||
}
|
||||
|
||||
// Exit.
|
||||
return result == cudaSuccess ? 0 : -1;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
# Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
# provided that the following conditions are met:
|
||||
# * Redistributions of source code must retain the above copyright notice, this list of
|
||||
# conditions and the following disclaimer.
|
||||
# * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
# conditions and the following disclaimer in the documentation and/or other materials
|
||||
# provided with the distribution.
|
||||
# * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
# to endorse or promote products derived from this software without specific prior written
|
||||
# permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
set(EXAMPLES_BASIC_CUTLASS_GEMM_SOURCES
|
||||
wmma_gemm.cu
|
||||
)
|
||||
|
||||
if (NOT CUTLASS_NATIVE_CUDA)
|
||||
# cuda_add_executable does not take interface include directories into account
|
||||
# Let's fetch them and pass them to CUDA.
|
||||
get_target_property(CUTLASS_INCLUDES CUTLASS INTERFACE_INCLUDE_DIRECTORIES)
|
||||
include_directories("${CUTLASS_INCLUDES}")
|
||||
endif()
|
||||
|
||||
cutlass_add_executable(
|
||||
05_wmma_gemm
|
||||
${EXAMPLES_BASIC_CUTLASS_GEMM_SOURCES}
|
||||
)
|
||||
@@ -0,0 +1,353 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*
|
||||
This example demonstrates how to call a CUTLASS GEMM kernel using Turing integer WMMA.
|
||||
|
||||
The CUTLASS integer WMMA Gemm template is instantiated in the function Cutlass_S8_WmmagemmNN. This
|
||||
is kernel computes the general matrix product (GEMM) using integer arithmetic accelerated by Turing
|
||||
WMMA and assumes all matrices have column-major layout.
|
||||
|
||||
The threadblock tile size is chosen as 128x128x8 which offers good performance for large matrices.
|
||||
See the CUTLASS Parallel for All blog post for more exposition on the tunable parameters available
|
||||
in CUTLASS.
|
||||
|
||||
https://devblogs.nvidia.com/cutlass-linear-algebra-cuda/
|
||||
|
||||
This example uses CUTLASS utilities to ease the matrix operations.
|
||||
*/
|
||||
|
||||
// Standard Library includes
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <vector>
|
||||
|
||||
// CUTLASS includes needed for WMMA GEMM kernel
|
||||
#include "cutlass/wmma_matrix.h"
|
||||
|
||||
// This example works only when this MACRO is defined in "cutlass/wmma_matrix.h"
|
||||
#ifdef CUTLASS_USE_SUBBYTE_WMMA
|
||||
|
||||
// Defines cutlass::gemm::Gemm, the generic Gemm computation template class.
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
|
||||
// Defines cutlass::gemm::WmmaGemmTraits, the structural components for WMMA GEMM
|
||||
#include "cutlass/gemm/wmma_gemm_traits.h"
|
||||
|
||||
//
|
||||
// CUTLASS utility includes
|
||||
//
|
||||
|
||||
// Defines operator<<() to write TensorView objects to std::ostream
|
||||
#include "tools/util/tensor_view_io.h"
|
||||
|
||||
// Defines cutlass::HostMatrix<>
|
||||
#include "tools/util/host_matrix.h"
|
||||
|
||||
// Defines cutlass::reference::device::TensorInitialize()
|
||||
#include "tools/util/reference/device/tensor_elementwise.h"
|
||||
|
||||
// Defines cutlass::reference::host::TensorEquals()
|
||||
#include "tools/util/reference/host/tensor_elementwise.h"
|
||||
|
||||
// Defines cutlass::reference::host::Gemm()
|
||||
#include "tools/util/reference/host/gemm.h"
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// This function defines a CUTLASS GEMM kernel instantiation, constructs its parameters object,
|
||||
// and launches it on the CUDA device.
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Define a CUTLASS GEMM template and launch a GEMM kernel.
|
||||
cudaError_t Cutlass_S8_WmmagemmNN(
|
||||
int M,
|
||||
int N,
|
||||
int K,
|
||||
int alpha,
|
||||
signed char const *A,
|
||||
int lda,
|
||||
signed char const *B,
|
||||
int ldb,
|
||||
int beta,
|
||||
int *C,
|
||||
int ldc) {
|
||||
|
||||
// Define type definition for 8-bit signed int WMMA CUTLASS GEMM with column-major
|
||||
// input matrices and 128x128x128 threadblock tile size.
|
||||
//
|
||||
// Note, A and B are 8-bit signed int. C and D are 32-bit int. .
|
||||
//
|
||||
typedef cutlass::gemm::WmmaGemmTraits<
|
||||
cutlass::MatrixLayout::kColumnMajor, // layout of A matrix
|
||||
cutlass::MatrixLayout::kColumnMajor, // layout of B matrix
|
||||
cutlass::Shape<128, 128, 128>, // threadblock tile size
|
||||
signed char, // A type
|
||||
signed char, // B type
|
||||
int, // D type
|
||||
cutlass::gemm::LinearScaling<int>, // functor to do the math in the epilogue
|
||||
int, // accumulator type
|
||||
cutlass::Shape<128, 32, 32>, // warp tile size
|
||||
cutlass::Shape<16, 16, 16>, // WMMA instruction tile size
|
||||
16, // scalars every time a thread loads from A
|
||||
16 // scalars every time a thread loads from B
|
||||
>
|
||||
GemmTraits;
|
||||
|
||||
// Define a CUTLASS GEMM type from a GemmTraits<> instantiation.
|
||||
typedef cutlass::gemm::Gemm<GemmTraits> Gemm;
|
||||
|
||||
// Construct and initialize CUTLASS GEMM parameters object.
|
||||
typename Gemm::Params params;
|
||||
|
||||
int result = params.initialize(
|
||||
M, // GEMM M dimension
|
||||
N, // GEMM N dimension
|
||||
K, // GEMM K dimension
|
||||
alpha, // scalar alpha
|
||||
A, // matrix A operand
|
||||
lda,
|
||||
B, // matrix B operand
|
||||
ldb,
|
||||
beta, // scalar beta
|
||||
C, // source matrix C
|
||||
ldc,
|
||||
C, // destination matrix C (may be different memory than source C matrix)
|
||||
ldc
|
||||
);
|
||||
|
||||
if (result) {
|
||||
std::cerr << "Failed to initialize CUTLASS Gemm::Params object." << std::endl;
|
||||
return cudaErrorInvalidValue;
|
||||
}
|
||||
|
||||
// Launch the CUTLASS GEMM kernel.
|
||||
Gemm::launch(params);
|
||||
|
||||
// Return any errors associated with the launch or cudaSuccess if no error.
|
||||
return cudaGetLastError();
|
||||
}
|
||||
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Allocate several matrices in GPU device memory and call an integer
|
||||
/// CUTLASS WMMA GEMM kernel.
|
||||
cudaError_t TestCutlassGemm(int M, int N, int K, int alpha, int beta) {
|
||||
cudaError_t result;
|
||||
|
||||
//
|
||||
// Construct cutlass::HostMatrix<> using the integer host-side types.
|
||||
|
||||
// M-by-K matrix of signed char
|
||||
cutlass::HostMatrix<signed char> A(cutlass::MatrixCoord(M, K));
|
||||
|
||||
// K-by-N matrix of signed char
|
||||
cutlass::HostMatrix<signed char> B(cutlass::MatrixCoord(K, N));
|
||||
|
||||
// M-by-N matrix of int
|
||||
cutlass::HostMatrix<int> C_cutlass(cutlass::MatrixCoord(M, N));
|
||||
|
||||
// M-by-N matrix of int
|
||||
cutlass::HostMatrix<int> C_reference(cutlass::MatrixCoord(M, N));
|
||||
|
||||
//
|
||||
// Initialize matrices with small, random integers.
|
||||
//
|
||||
|
||||
cutlass::Distribution dist;
|
||||
|
||||
// Uniform random distribution from -4 .. 4. Values are truncated to integers.
|
||||
dist.set_uniform(-4, 4);
|
||||
|
||||
// Arbitrary RNG seed value. Hard-coded for deterministic results.
|
||||
int seed = 2080;
|
||||
|
||||
cutlass::reference::device::TensorInitialize(
|
||||
A.device_view(), // concept: TensorView
|
||||
seed,
|
||||
dist);
|
||||
|
||||
cutlass::reference::device::TensorInitialize(
|
||||
B.device_view(), // concept: TensorView
|
||||
seed * 2,
|
||||
dist);
|
||||
|
||||
cutlass::reference::device::TensorInitialize(
|
||||
C_cutlass.device_view(), // concept: TensorView
|
||||
seed * 3,
|
||||
dist);
|
||||
|
||||
// Copy C_cutlass into C_reference so the GEMM is correct when beta != 0.
|
||||
cutlass::reference::device::TensorFill(C_reference.device_view(), C_cutlass.device_view());
|
||||
|
||||
// Copy the device-side view into host memory
|
||||
C_reference.sync_host();
|
||||
|
||||
//
|
||||
// Launch the CUTLASS GEMM kernel
|
||||
//
|
||||
|
||||
result = Cutlass_S8_WmmagemmNN(
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
alpha,
|
||||
A.device_data(),
|
||||
A.leading_dim(),
|
||||
B.device_data(),
|
||||
B.leading_dim(),
|
||||
beta,
|
||||
C_cutlass.device_data(),
|
||||
C_cutlass.leading_dim()
|
||||
);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
return result;
|
||||
}
|
||||
|
||||
//
|
||||
// Verify the result using a host-side reference
|
||||
//
|
||||
|
||||
// A and B were initialized using device-side procedures.
|
||||
A.sync_host();
|
||||
B.sync_host();
|
||||
|
||||
// Copy CUTLASS's GEMM results into host memory.
|
||||
C_cutlass.sync_host();
|
||||
|
||||
// Compute the reference result using the host-side GEMM reference implementation.
|
||||
cutlass::reference::host::Gemm(
|
||||
cutlass::gemm::GemmCoord(K, N, M), // problem size (type: cutlass::gemm::GemmCoord)
|
||||
alpha, // alpha (type: int)
|
||||
A.host_ref(), // A (concept: TensorRef)
|
||||
B.host_ref(), // B (concept: TensorRef)
|
||||
beta, // beta (int)
|
||||
C_reference.host_ref(), // C (concept: TensorRef)
|
||||
int(0) // Accumulator initial value passed as argument to deduce
|
||||
); // internal accumulation data type as int.
|
||||
|
||||
// Compare reference to computed results.
|
||||
if (!cutlass::reference::host::TensorEquals(C_reference.host_view(), C_cutlass.host_view())) {
|
||||
|
||||
std::cerr << "Error - CUTLASS WMMA GEMM kernel differs from reference." << std::endl;
|
||||
|
||||
//
|
||||
// On error, print C_cutlass and C_reference to std::cerr.
|
||||
//
|
||||
|
||||
// Result of CUTLASS WMMA GEMM kernel
|
||||
std::cerr << "CUTLASS:\n" << C_cutlass << std::endl;
|
||||
|
||||
// Result of reference computation
|
||||
std::cerr << "Reference:\n" << C_reference << std::endl;
|
||||
|
||||
// Return error code.
|
||||
return cudaErrorUnknown;
|
||||
}
|
||||
|
||||
// Passed error check
|
||||
return cudaSuccess;
|
||||
}
|
||||
#endif // defined CUTLASS_USE_SUBBYTE_WMMA
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Entry point to wmma_gemm example.
|
||||
//
|
||||
// usage:
|
||||
//
|
||||
// 05_wmma_gemm <M> <N> <K> <alpha> <beta>
|
||||
//
|
||||
int main(int argc, const char *arg[]) {
|
||||
|
||||
#ifdef CUTLASS_USE_SUBBYTE_WMMA
|
||||
// Properties of CUDA device
|
||||
cudaDeviceProp device_properties;
|
||||
|
||||
// Assumne the device id is 0.
|
||||
int device_id = 0;
|
||||
|
||||
cudaError_t result = cudaGetDeviceProperties(&device_properties, device_id);
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Failed to get device properties: "
|
||||
<< cudaGetErrorString(result) << std::endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
if ((device_properties.major * 10 + device_properties.minor) < 75) {
|
||||
std::cerr << "This example needs to run on a Turing device." << std::endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
//
|
||||
// Parse the command line to obtain GEMM dimensions and scalar values.
|
||||
//
|
||||
|
||||
// GEMM problem dimensions.
|
||||
int problem[3] = { 128, 128, 128 };
|
||||
|
||||
for (int i = 1; i < argc && i < 4; ++i) {
|
||||
std::stringstream ss(arg[i]);
|
||||
ss >> problem[i - 1];
|
||||
}
|
||||
|
||||
// Scalars used for linear scaling the result of the matrix product.
|
||||
int scalars[2] = { 1, 0 };
|
||||
|
||||
for (int i = 4; i < argc && i < 6; ++i) {
|
||||
std::stringstream ss(arg[i]);
|
||||
ss >> scalars[i - 4];
|
||||
}
|
||||
|
||||
//
|
||||
// Run the CUTLASS GEMM test.
|
||||
//
|
||||
|
||||
result = TestCutlassGemm(
|
||||
problem[0], // GEMM M dimension
|
||||
problem[1], // GEMM N dimension
|
||||
problem[2], // GEMM K dimension
|
||||
scalars[0], // alpha
|
||||
scalars[1] // beta
|
||||
);
|
||||
|
||||
if (result == cudaSuccess) {
|
||||
std::cout << "Passed." << std::endl;
|
||||
}
|
||||
|
||||
// Exit.
|
||||
return result == cudaSuccess ? 0 : -1;
|
||||
|
||||
#else
|
||||
std::cerr << "CUTLASS WMMA GEMM targeting Turing Tensor Cores features requires CUDA 10." << std::endl;
|
||||
return -1;
|
||||
#endif // defined CUTLASS_USE_SUBBYTE_WMMA
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
# Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
# provided that the following conditions are met:
|
||||
# * Redistributions of source code must retain the above copyright notice, this list of
|
||||
# conditions and the following disclaimer.
|
||||
# * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
# conditions and the following disclaimer in the documentation and/or other materials
|
||||
# provided with the distribution.
|
||||
# * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
# to endorse or promote products derived from this software without specific prior written
|
||||
# permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
add_subdirectory(00_basic_gemm)
|
||||
add_subdirectory(01_tensor_view)
|
||||
add_subdirectory(02_cutlass_utilities)
|
||||
add_subdirectory(03_strided_batched_gemm)
|
||||
add_subdirectory(04_tile_iterator)
|
||||
add_subdirectory(05_wmma_gemm)
|
||||
Reference in New Issue
Block a user