Committing CUTLASS for release.
This commit is contained in:
akerr
@@ -0,0 +1,7 @@
|
||||
/bin/
|
||||
/gemm-GPU.csv
|
||||
/gemm-REF.csv
|
||||
/a.csv
|
||||
/b.csv
|
||||
/gp100_schmoo/
|
||||
/ignore/
|
||||
@@ -0,0 +1,213 @@
|
||||
#/******************************************************************************
|
||||
# * Copyright (c) 2011-2017, NVIDIA CORPORATION. All rights reserved.
|
||||
# *
|
||||
# * Redistribution and use in source and binary forms, with or without
|
||||
# * modification, are not permitted.
|
||||
# *
|
||||
# * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
# * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
# * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
|
||||
# * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
# * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
# * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
# * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
# * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
# *
|
||||
#******************************************************************************/
|
||||
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
#
|
||||
# Makefile usage
|
||||
#
|
||||
# make <target> sm=<XX[,YY,ZZ,..]> [transpose=<nn*|nt|tn|tt>] [verbose=<0*|1>] [keep=<0*|1>]
|
||||
#
|
||||
# * : default
|
||||
#
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
TEST_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
|
||||
|
||||
include ../common.mk
|
||||
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Commandline Options
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
ifdef transpose
|
||||
TRANSPOSE := $(transpose)
|
||||
else
|
||||
TRANSPOSE := nn
|
||||
endif
|
||||
|
||||
ifdef deepbench
|
||||
BENCHMARK_DEEPBENCH := $(deepbench)
|
||||
else
|
||||
BENCHMARK_DEEPBENCH := 0
|
||||
endif
|
||||
|
||||
# If defined, GEMMs only compiled with specified alignment restrictions on A and B
|
||||
# matrices. Otherwise, kernels are compiled for all feasible alignment options, and
|
||||
# the appropriate kernel is selected.
|
||||
ifdef alignment
|
||||
DEFINES += -DGEMM_ALIGNMENT=$(alignment)
|
||||
endif
|
||||
|
||||
# If defined as false, ragged handling can be disabled.
|
||||
ifdef ragged
|
||||
DEFINES += -DGEMM_RAGGED=$(ragged)
|
||||
endif
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Include and Library paths
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
INC += -I$(TEST_DIR)
|
||||
INC += -I$(BASE_DIR)
|
||||
|
||||
LIBS += -lcublas
|
||||
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Preprocessor definitions
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
ifeq (nt, $(TRANSPOSE))
|
||||
DEFINES += -DTRANSPOSE_B
|
||||
|
||||
else ifeq (tn, $(TRANSPOSE))
|
||||
DEFINES += -DTRANSPOSE_A
|
||||
|
||||
else ifeq (tt, $(TRANSPOSE))
|
||||
DEFINES += -DTRANSPOSE_A
|
||||
DEFINES += -DTRANSPOSE_B
|
||||
endif
|
||||
|
||||
NVCCFLAGS += -std=c++11
|
||||
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Dependency Lists
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
DEPS := $(call rwildcard, $(BASE_DIR),*.h) \
|
||||
$(call rwildcard, $(BASE_DIR)cgl,*.h) \
|
||||
$(BASE_DIR)common.mk \
|
||||
$(TEST_DIR)Makefile
|
||||
|
||||
|
||||
ALL := sgemm \
|
||||
dgemm \
|
||||
hgemm \
|
||||
igemm
|
||||
|
||||
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# make default
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
default:
|
||||
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# make clean
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
clean :
|
||||
rm -f bin/*
|
||||
rm -f *.i* *.cubin *.cu.c *.cudafe* *.fatbin.c *.ptx *.hash *.cu.cpp *.o *.obj* *dlink.* *.res *.fatbin *.module_id
|
||||
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# make all
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
all : $(ALL)
|
||||
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# make sgemm
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
sgemm: bin/sgemm_$(TRANSPOSE)_$(BIN_SUFFIX)
|
||||
|
||||
sgemm_testbench: bin/sgemm_testbench_$(BIN_SUFFIX)
|
||||
|
||||
bin/sgemm_$(TRANSPOSE)_$(BIN_SUFFIX) : gemm.cu $(DEPS)
|
||||
mkdir -p bin
|
||||
$(NVCC) -DTEST_SGEMM $(DEFINES) $(SM_TARGETS) -o $@ gemm.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBINC) $(LIBS)
|
||||
|
||||
bin/sgemm_testbench_$(BIN_SUFFIX) : gemm_testbench.cu $(DEPS)
|
||||
mkdir -p bin
|
||||
$(NVCC) -DTEST_SGEMM $(DEFINES) $(SM_TARGETS) -D BENCHMARK_DEEPBENCH=$(BENCHMARK_DEEPBENCH) -o $@ gemm_testbench.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBINC) $(LIBS)
|
||||
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# make dgemm
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
dgemm: bin/dgemm_$(TRANSPOSE)_$(BIN_SUFFIX)
|
||||
|
||||
dgemm_testbench: bin/dgemm_testbench_$(BIN_SUFFIX)
|
||||
|
||||
bin/dgemm_$(TRANSPOSE)_$(BIN_SUFFIX) : gemm.cu $(DEPS)
|
||||
mkdir -p bin
|
||||
$(NVCC) -DTEST_DGEMM $(DEFINES) $(SM_TARGETS) -o $@ gemm.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBINC) $(LIBS)
|
||||
|
||||
bin/dgemm_testbench_$(BIN_SUFFIX) : gemm_testbench.cu $(DEPS)
|
||||
mkdir -p bin
|
||||
$(NVCC) -DTEST_DGEMM $(DEFINES) $(SM_TARGETS) -D BENCHMARK_DEEPBENCH=$(BENCHMARK_DEEPBENCH) -o $@ gemm_testbench.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBINC) $(LIBS)
|
||||
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# make hgemm
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
hgemm: bin/hgemm_$(TRANSPOSE)_$(BIN_SUFFIX)
|
||||
|
||||
hgemm_testbench: bin/hgemm_testbench_$(BIN_SUFFIX)
|
||||
|
||||
bin/hgemm_$(TRANSPOSE)_$(BIN_SUFFIX) : gemm.cu $(DEPS)
|
||||
mkdir -p bin
|
||||
$(NVCC) -DTEST_HGEMM $(DEFINES) $(SM_TARGETS) -o $@ gemm.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBINC) $(LIBS)
|
||||
|
||||
bin/hgemm_testbench_$(BIN_SUFFIX) : gemm_testbench.cu $(DEPS)
|
||||
mkdir -p bin
|
||||
$(NVCC) -DTEST_HGEMM $(DEFINES) $(SM_TARGETS) -D BENCHMARK_DEEPBENCH=$(BENCHMARK_DEEPBENCH) -o $@ gemm_testbench.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBINC) $(LIBS)
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# make igemm
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
igemm: bin/igemm_$(TRANSPOSE)_$(BIN_SUFFIX)
|
||||
|
||||
bin/igemm_$(TRANSPOSE)_$(BIN_SUFFIX) : gemm.cu $(DEPS)
|
||||
mkdir -p bin
|
||||
$(NVCC) -DTEST_IGEMM $(DEFINES) $(SM_TARGETS) -o $@ gemm.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBINC) $(LIBS)
|
||||
|
||||
igemm_testbench: bin/igemm_testbench_$(BIN_SUFFIX)
|
||||
|
||||
bin/igemm_testbench_$(BIN_SUFFIX) : gemm_testbench.cu $(DEPS)
|
||||
mkdir -p bin
|
||||
$(NVCC) -DTEST_IGEMM $(DEFINES) $(SM_TARGETS) -D BENCHMARK_DEEPBENCH=$(BENCHMARK_DEEPBENCH) -o $@ gemm_testbench.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBINC) $(LIBS)
|
||||
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# make wgemm
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
wgemm: bin/wgemm_$(TRANSPOSE)_$(BIN_SUFFIX)
|
||||
|
||||
wgemm_testbench: bin/wgemm_testbench_$(BIN_SUFFIX)
|
||||
|
||||
bin/wgemm_$(TRANSPOSE)_$(BIN_SUFFIX) : gemm.cu $(DEPS)
|
||||
mkdir -p bin
|
||||
$(NVCC) -DTEST_WGEMM -DWMMA $(DEFINES) $(SM_TARGETS) -o $@ gemm.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBINC) $(LIBS)
|
||||
|
||||
bin/wgemm_testbench_$(BIN_SUFFIX) : gemm_testbench.cu $(DEPS)
|
||||
mkdir -p bin
|
||||
$(NVCC) -DTEST_WGEMM -DWMMA $(DEFINES) $(SM_TARGETS) -D BENCHMARK_DEEPBENCH=$(BENCHMARK_DEEPBENCH) -o $@ gemm_testbench.cu $(NVCCFLAGS) $(CPU_ARCH) $(INC) $(LIBINC) $(LIBS)
|
||||
@@ -0,0 +1,292 @@
|
||||
/******************************************************************************
|
||||
* Copyright (c) 2011-2017, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are not permitted.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
******************************************************************************/
|
||||
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* \file
|
||||
* C++ interface for dispatching CUBLAS GEMM calls
|
||||
*/
|
||||
|
||||
#include <cublas_v2.h>
|
||||
|
||||
namespace cutlass {
|
||||
|
||||
|
||||
/******************************************************************************
|
||||
* cuBLAS dispatch entrypoints
|
||||
******************************************************************************/
|
||||
|
||||
/**
|
||||
* Dispatch cuBLAS igemm
|
||||
*/
|
||||
cublasStatus_t cublas_gemm_dispatch(
|
||||
cublasHandle_t cublas_handle, ///< CUBLAS handle
|
||||
cublasOperation_t transform_a, ///< Transform op(A) that is non- or (conj.) transpose.
|
||||
cublasOperation_t transform_b, ///< Transform op(B) that is non- or (conj.) transpose.
|
||||
int m, ///< Height in rows of op(A) and C
|
||||
int n, ///< Width in columns of op(B) and C
|
||||
int k, ///< Width in columns of op(A) and height in rows of op(B)
|
||||
int32_t alpha, ///< Scalar used for multiplicands
|
||||
int8_t *d_a, ///< Device pointer to matrix A array values
|
||||
int8_t *d_b, ///< Device pointer to matrix B array values
|
||||
int32_t beta, ///< Scalar used for addend
|
||||
int32_t *d_c, ///< Device pointer to matrix C array values
|
||||
cudaStream_t stream = 0, ///< CUDA stream to launch kernels within. Default is stream<sub>0</sub>.
|
||||
bool debug_synchronous = false) ///< Whether or not to synchronize the stream after every kernel launch to check for errors.
|
||||
{
|
||||
return cublasGemmEx(
|
||||
cublas_handle,
|
||||
transform_a,
|
||||
transform_b,
|
||||
m,
|
||||
n,
|
||||
k,
|
||||
(void*) &alpha,
|
||||
(void*) d_a,
|
||||
CUDA_R_8I,
|
||||
(transform_a == CUBLAS_OP_N) ? m : k,
|
||||
(void*) d_b,
|
||||
CUDA_R_8I,
|
||||
(transform_b == CUBLAS_OP_N) ? k : n,
|
||||
(void*) &beta,
|
||||
(void*) d_c,
|
||||
CUDA_R_32I,
|
||||
m,
|
||||
CUDA_R_32I,
|
||||
CUBLAS_GEMM_DFALT);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Dispatch cuBLAS hgemm
|
||||
*/
|
||||
cublasStatus_t cublas_gemm_dispatch(
|
||||
cublasHandle_t cublas_handle, ///< CUBLAS handle
|
||||
cublasOperation_t transform_a, ///< Transform op(A) that is non- or (conj.) transpose.
|
||||
cublasOperation_t transform_b, ///< Transform op(B) that is non- or (conj.) transpose.
|
||||
int m, ///< Height in rows of op(A) and C
|
||||
int n, ///< Width in columns of op(B) and C
|
||||
int k, ///< Width in columns of op(A) and height in rows of op(B)
|
||||
__half alpha, ///< Scalar used for multiplicands
|
||||
__half *d_a, ///< Device pointer to matrix A array values
|
||||
__half *d_b, ///< Device pointer to matrix B array values
|
||||
__half beta, ///< Scalar used for addend
|
||||
__half *d_c, ///< Device pointer to matrix C array values
|
||||
cudaStream_t stream = 0, ///< CUDA stream to launch kernels within. Default is stream<sub>0</sub>.
|
||||
bool debug_synchronous = false) ///< Whether or not to synchronize the stream after every kernel launch to check for errors.
|
||||
{
|
||||
return cublasHgemm(
|
||||
cublas_handle, transform_a, transform_b,
|
||||
m, n, k,
|
||||
&alpha,
|
||||
d_a,
|
||||
(transform_a == CUBLAS_OP_N) ? m : k,
|
||||
d_b,
|
||||
(transform_b == CUBLAS_OP_N) ? k : n,
|
||||
&beta,
|
||||
d_c,
|
||||
m);
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Dispatch cuBLAS sgemm
|
||||
*/
|
||||
cublasStatus_t cublas_gemm_dispatch(
|
||||
cublasHandle_t cublas_handle, ///< CUBLAS handle
|
||||
cublasOperation_t transform_a, ///< Transform op(A) that is non- or (conj.) transpose.
|
||||
cublasOperation_t transform_b, ///< Transform op(B) that is non- or (conj.) transpose.
|
||||
int m, ///< Height in rows of op(A) and C
|
||||
int n, ///< Width in columns of op(B) and C
|
||||
int k, ///< Width in columns of op(A) and height in rows of op(B)
|
||||
float alpha, ///< Scalar used for multiplicands
|
||||
float *d_a, ///< Device pointer to matrix A array values
|
||||
float *d_b, ///< Device pointer to matrix B array values
|
||||
float beta, ///< Scalar used for addend
|
||||
float *d_c, ///< Device pointer to matrix C array values
|
||||
cudaStream_t stream = 0, ///< CUDA stream to launch kernels within. Default is stream<sub>0</sub>.
|
||||
bool debug_synchronous = false) ///< Whether or not to synchronize the stream after every kernel launch to check for errors.
|
||||
{
|
||||
return cublasSgemm(
|
||||
cublas_handle, transform_a, transform_b,
|
||||
m, n, k,
|
||||
&alpha,
|
||||
d_a,
|
||||
(transform_a == CUBLAS_OP_N) ? m : k,
|
||||
d_b,
|
||||
(transform_b == CUBLAS_OP_N) ? k : n,
|
||||
&beta,
|
||||
d_c,
|
||||
m);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Dispatch cuBLAS dgemm
|
||||
*/
|
||||
cublasStatus_t cublas_gemm_dispatch(
|
||||
cublasHandle_t cublas_handle, ///< CUBLAS handle
|
||||
cublasOperation_t transform_a, ///< Transform op(A) that is non- or (conj.) transpose.
|
||||
cublasOperation_t transform_b, ///< Transform op(B) that is non- or (conj.) transpose.
|
||||
int m, ///< Height in rows of op(A) and C
|
||||
int n, ///< Width in columns of op(B) and C
|
||||
int k, ///< Width in columns of op(A) and height in rows of op(B)
|
||||
double alpha, ///< Scalar used for multiplicands
|
||||
double *d_a, ///< Device pointer to matrix A array values
|
||||
double *d_b, ///< Device pointer to matrix B array values
|
||||
double beta, ///< Scalar used for addend
|
||||
double *d_c, ///< Device pointer to matrix C array values
|
||||
cudaStream_t stream = 0, ///< CUDA stream to launch kernels within. Default is stream<sub>0</sub>.
|
||||
bool debug_synchronous = false) ///< Whether or not to synchronize the stream after every kernel launch to check for errors.
|
||||
{
|
||||
return cublasDgemm(
|
||||
cublas_handle, transform_a, transform_b,
|
||||
m, n, k,
|
||||
&alpha,
|
||||
d_a, (transform_a == CUBLAS_OP_N) ? m : k,
|
||||
d_b, (transform_b == CUBLAS_OP_N) ? k : n,
|
||||
&beta,
|
||||
d_c, m);
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch cuBLAS Tensor Cores GEMM
|
||||
*/
|
||||
cublasStatus_t cublas_gemm_dispatch(
|
||||
cublasHandle_t cublas_handle, ///< CUBLAS handle
|
||||
cublasOperation_t transform_a, ///< Transform op(A) that is non- or (conj.) transpose.
|
||||
cublasOperation_t transform_b, ///< Transform op(B) that is non- or (conj.) transpose.
|
||||
int m, ///< Height in rows of op(A) and C
|
||||
int n, ///< Width in columns of op(B) and C
|
||||
int k, ///< Width in columns of op(A) and height in rows of op(B)
|
||||
float alpha, ///< Scalar used for multiplicands
|
||||
half *d_a, ///< Device pointer to matrix A array values
|
||||
half *d_b, ///< Device pointer to matrix B array values
|
||||
float beta, ///< Scalar used for addend
|
||||
float *d_c, ///< Device pointer to matrix C array values
|
||||
cudaStream_t stream = 0, ///< CUDA stream to launch kernels within. Default is stream<sub>0</sub>.
|
||||
bool debug_synchronous = false) ///< Whether or not to synchronize the stream after every kernel launch to check for errors.
|
||||
{
|
||||
return cublasGemmEx(
|
||||
cublas_handle,
|
||||
transform_a,
|
||||
transform_b,
|
||||
m,
|
||||
n,
|
||||
k,
|
||||
(void*) &alpha,
|
||||
(void*) d_a,
|
||||
CUDA_R_16F,
|
||||
(transform_a == CUBLAS_OP_N) ? m : k,
|
||||
(void*) d_b,
|
||||
CUDA_R_16F,
|
||||
(transform_b == CUBLAS_OP_N) ? k : n,
|
||||
(void*) &beta,
|
||||
(void*) d_c,
|
||||
CUDA_R_32F,
|
||||
m,
|
||||
CUDA_R_32F,
|
||||
CUBLAS_GEMM_DFALT_TENSOR_OP);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Uses cuBLAS to compute gemm on device matrices (unspecialized)
|
||||
*/
|
||||
template <
|
||||
gemm::tiling_strategy::kind_t _TilingStrategy, ///< Tile-sizing classification category
|
||||
math_operation_class_t _math_op,
|
||||
matrix_transform_t::kind_t _TransformA, ///< Transformation op for matrix A
|
||||
matrix_transform_t::kind_t _TransformB, ///< Transformation op for matrix B
|
||||
typename _value, ///< Multiplicand value type (matrices A and B)
|
||||
typename _accum ///< Accumulator value type (matrix C and scalars)
|
||||
>
|
||||
struct cublas_gemm
|
||||
{
|
||||
//
|
||||
// Type alias definitions
|
||||
//
|
||||
|
||||
static const gemm::tiling_strategy::kind_t TilingStrategy = _TilingStrategy;
|
||||
static const math_operation_class_t math_op = _math_op;
|
||||
static const matrix_transform_t::kind_t TransformA = _TransformA;
|
||||
static const matrix_transform_t::kind_t TransformB = _TransformB;
|
||||
|
||||
using value_t = _value;
|
||||
using accum_t = _accum;
|
||||
|
||||
/// Launches a GEMM
|
||||
gemm::launch_configuration operator()(
|
||||
cublasHandle_t cublas_handle, ///< CUBLAS handle
|
||||
int m,
|
||||
int n,
|
||||
int k,
|
||||
value_t *A, ///< A matrix
|
||||
value_t *B, ///< B matrix
|
||||
accum_t *C, ///< C matrix
|
||||
accum_t alpha, ///< Scalar used for multiplicands
|
||||
accum_t beta, ///< Scalar used for addend
|
||||
cudaStream_t stream = 0, ///< CUDA stream to launch kernels within. Default is stream<sub>0</sub>.
|
||||
bool debug_synchronous = false) ///< Whether or not to synchronize the stream after every kernel launch to check for errors.
|
||||
{
|
||||
cublasStatus_t cublas_error = cublas_gemm_dispatch(
|
||||
cublas_handle,
|
||||
(cublasOperation_t) TransformA,
|
||||
(cublasOperation_t) TransformB,
|
||||
m,
|
||||
n,
|
||||
k,
|
||||
alpha,
|
||||
A,
|
||||
B,
|
||||
beta,
|
||||
C,
|
||||
stream,
|
||||
debug_synchronous);
|
||||
|
||||
cudaError_t error;
|
||||
if (cublas_error != CUBLAS_STATUS_SUCCESS)
|
||||
{
|
||||
if (cublas_error == CUBLAS_STATUS_NOT_SUPPORTED) {
|
||||
return gemm::launch_configuration(cudaErrorInvalidValue);
|
||||
}
|
||||
|
||||
error = cudaGetLastError();
|
||||
if (error == cudaSuccess) {
|
||||
return gemm::launch_configuration(cudaErrorUnknown);
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
// Check for failure to launch
|
||||
if (CUDA_PERROR_DEBUG(error = cudaPeekAtLastError()))
|
||||
return gemm::launch_configuration(error);
|
||||
|
||||
// Sync the stream if specified to flush runtime errors
|
||||
if (debug_synchronous && (CUDA_PERROR_DEBUG(error = cudaStreamSynchronize(stream))))
|
||||
return gemm::launch_configuration(error);
|
||||
|
||||
return gemm::launch_configuration(error);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
} // namespace cutlass
|
||||
@@ -0,0 +1,253 @@
|
||||
/******************************************************************************
|
||||
* Copyright (c) 2011-2017, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are not permitted.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
******************************************************************************/
|
||||
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* \file Dispatch routines for CUTLASS GEMM kernels
|
||||
*/
|
||||
|
||||
// CUDA includes
|
||||
#include <cublas_v2.h>
|
||||
|
||||
// Cutlass GEMM API
|
||||
#include <cutlass/util/util.h>
|
||||
#include <cutlass/gemm/dispatch.h>
|
||||
#include <cutlass/gemm/epilogue_function.h>
|
||||
|
||||
// Test utilities
|
||||
#include "util/type_conversion.h"
|
||||
|
||||
namespace cutlass {
|
||||
|
||||
|
||||
|
||||
/******************************************************************************
|
||||
* Cutlass dispatch entrypoints
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Compile-time overrides for alignment and ragged handling.
|
||||
//
|
||||
|
||||
// If zero, all feasible alignment options are supported.
|
||||
#ifndef GEMM_ALIGNMENT
|
||||
#define GEMM_ALIGNMENT 0
|
||||
#endif
|
||||
|
||||
// If true, kernels are compiled with ragged handling enabled.
|
||||
#ifndef GEMM_RAGGED
|
||||
#define GEMM_RAGGED true
|
||||
#endif
|
||||
|
||||
//
|
||||
// Dispatch logic given problem size specialization, math operation class, layout
|
||||
// and type of operands, and epilogue operation.
|
||||
//
|
||||
|
||||
/**
|
||||
* Cutlass GEMM dispatch
|
||||
*/
|
||||
template <
|
||||
gemm::tiling_strategy::kind_t _TilingStrategy, ///< Tile-sizing classification category
|
||||
math_operation_class_t _math_op, // Indicates
|
||||
matrix_transform_t::kind_t _TransformA, ///< Transformation op for matrix A
|
||||
matrix_transform_t::kind_t _TransformB, ///< Transformation op for matrix B
|
||||
typename _value, ///< Multiplicand value type (matrices A and B)
|
||||
typename _accum, ///< Accumulator value type (matrix C and scalars)
|
||||
typename _epilogue_op_t ///< Epilogue opeartion to update matrix C
|
||||
= gemm::blas_scaled_epilogue<_accum, _accum, _accum>
|
||||
>
|
||||
struct cutlass_gemm_dispatch
|
||||
{
|
||||
//
|
||||
// Type alias definitions
|
||||
//
|
||||
|
||||
static const gemm::tiling_strategy::kind_t TilingStrategy = _TilingStrategy;
|
||||
static const math_operation_class_t math_op = _math_op;
|
||||
static const matrix_transform_t::kind_t TransformA = _TransformA;
|
||||
static const matrix_transform_t::kind_t TransformB = _TransformB;
|
||||
|
||||
using value_t = _value;
|
||||
using accum_t = _accum;
|
||||
using epilogue_op_t = _epilogue_op_t;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Returns leading dimension for A matrix operand
|
||||
int leading_dim_a(int m, int k) const
|
||||
{
|
||||
return (TransformA == matrix_transform_t::NonTranspose ? m : k);
|
||||
}
|
||||
|
||||
/// Returns leading dimension for B matrix operand
|
||||
int leading_dim_b(int k, int n) const
|
||||
{
|
||||
return (TransformB == matrix_transform_t::NonTranspose ? k : n);
|
||||
}
|
||||
|
||||
/// Launches a GEMM
|
||||
template <int operand_alignment, int accumulator_alignment>
|
||||
gemm::launch_configuration launch(
|
||||
int m,
|
||||
int n,
|
||||
int k,
|
||||
epilogue_op_t epilogue_op,
|
||||
value_t *A,
|
||||
value_t *B,
|
||||
accum_t *C,
|
||||
cudaStream_t stream = 0,
|
||||
bool debug_synchronous = false)
|
||||
{
|
||||
return gemm::device_gemm<
|
||||
TilingStrategy,
|
||||
math_op,
|
||||
TransformA,
|
||||
operand_alignment,
|
||||
TransformB,
|
||||
operand_alignment,
|
||||
value_t,
|
||||
accum_t,
|
||||
epilogue_op_t,
|
||||
accumulator_alignment>
|
||||
(
|
||||
m,
|
||||
n,
|
||||
k,
|
||||
epilogue_op,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
stream,
|
||||
debug_synchronous);
|
||||
}
|
||||
|
||||
/// Dispatches a CUTLASS GEMM
|
||||
gemm::launch_configuration operator()(
|
||||
cublasHandle_t handle, ///< CUBLAS handle
|
||||
int m, ///< Rows of GEMM problem
|
||||
int n, ///< Columns of GEMM problem
|
||||
int k, ///< Inner dimension of GEMM problem
|
||||
value_t *A, ///< A matrix
|
||||
value_t *B, ///< B matrix
|
||||
accum_t *C, ///< C matrix
|
||||
accum_t alpha, ///< Scalar used for multiplicands
|
||||
accum_t beta, ///< Scalar used for addend
|
||||
cudaStream_t stream = 0, ///< CUDA stream to launch kernels within.
|
||||
bool debug_synchronous = false) ///< Whether or not to synchronize the stream
|
||||
/// after every kernel launch to check for errors.
|
||||
{
|
||||
|
||||
// Forces kernel selection to choose specific alignment (in bytes)
|
||||
int const force_operand_alignment = GEMM_ALIGNMENT;
|
||||
|
||||
// Problem size must be multiple of the smallest vector load size
|
||||
typedef value_t operand_load_t;
|
||||
int const accumulator_alignment = sizeof(accum_t);
|
||||
|
||||
int const lda = leading_dim_a(m, k);
|
||||
int const ldb = leading_dim_b(k, n);
|
||||
|
||||
epilogue_op_t epilogue(alpha, beta);
|
||||
|
||||
// TODO: opportunity for metaprogramming loop
|
||||
|
||||
// Prefer the largest granularity of vector load that is compatible with
|
||||
// problem size and data alignment.
|
||||
if ((!force_operand_alignment || force_operand_alignment == 16) &&
|
||||
!((sizeof(operand_load_t) * lda) % 16) &&
|
||||
!((sizeof(operand_load_t) * ldb) % 16))
|
||||
{
|
||||
#if !(GEMM_ALIGNMENT) || (GEMM_ALIGNMENT == 16)
|
||||
return launch<__NV_STD_MAX(16, sizeof(value_t)), accumulator_alignment>(
|
||||
m,
|
||||
n,
|
||||
k,
|
||||
epilogue,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
stream,
|
||||
debug_synchronous);
|
||||
#endif
|
||||
}
|
||||
else if ((!force_operand_alignment || force_operand_alignment == 8) &&
|
||||
!((sizeof(operand_load_t) * lda) % 8) &&
|
||||
!((sizeof(operand_load_t) * ldb) % 8))
|
||||
{
|
||||
#if !(GEMM_ALIGNMENT) || (GEMM_ALIGNMENT == 8)
|
||||
return launch<__NV_STD_MAX(8, sizeof(value_t)), accumulator_alignment>(
|
||||
m,
|
||||
n,
|
||||
k,
|
||||
epilogue,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
stream,
|
||||
debug_synchronous);
|
||||
#endif
|
||||
}
|
||||
else if ((!force_operand_alignment || force_operand_alignment == 4) &&
|
||||
!((sizeof(operand_load_t) * lda) % 4) &&
|
||||
!((sizeof(operand_load_t) * ldb) % 4))
|
||||
{
|
||||
#if !(GEMM_ALIGNMENT) || (GEMM_ALIGNMENT == 4)
|
||||
return launch<__NV_STD_MAX(4, sizeof(value_t)), accumulator_alignment>(
|
||||
m,
|
||||
n,
|
||||
k,
|
||||
epilogue,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
stream,
|
||||
debug_synchronous);
|
||||
#endif
|
||||
}
|
||||
else if ((!force_operand_alignment || force_operand_alignment == 2) &&
|
||||
!((sizeof(operand_load_t) * lda) % 2) &&
|
||||
!((sizeof(operand_load_t) * ldb) % 2))
|
||||
{
|
||||
// 16-bit alignment only supported for HGEMM
|
||||
#if defined(TEST_HGEMM) || defined(TEST_WGEMM)
|
||||
#if !(GEMM_ALIGNMENT) || (GEMM_ALIGNMENT == 2)
|
||||
return launch<__NV_STD_MAX(2, sizeof(value_t)), accumulator_alignment>(
|
||||
m,
|
||||
n,
|
||||
k,
|
||||
epilogue,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
stream,
|
||||
debug_synchronous);
|
||||
#endif
|
||||
#endif
|
||||
}
|
||||
|
||||
return gemm::launch_configuration(cudaErrorInvalidValue);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
} // namespace cutlass
|
||||
@@ -0,0 +1,564 @@
|
||||
/******************************************************************************
|
||||
* Copyright (c) 2011-2017, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are not permitted.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
******************************************************************************/
|
||||
|
||||
/**
|
||||
* \file gemm.cu
|
||||
* GEMM test driver
|
||||
*
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
#include <typeinfo>
|
||||
#include <random>
|
||||
#include <stdint.h>
|
||||
|
||||
// CUBLAS GEMM API
|
||||
#include <cublas_v2.h>
|
||||
|
||||
// Set Cutlass debug macro to enable console printing of library errors
|
||||
#define DEBUG
|
||||
|
||||
#if defined(WMMA)
|
||||
// Conditionally include WMMA headers (CUDA 9 Preview Feature)
|
||||
#include <mma.h>
|
||||
#endif
|
||||
|
||||
// Cutlass GEMM API
|
||||
#include <cutlass/util/util.h>
|
||||
#include <cutlass/gemm/dispatch.h>
|
||||
#include <cutlass/gemm/epilogue_function.h>
|
||||
|
||||
// Test utilities
|
||||
#include "util/command_line.h"
|
||||
#include "util/half.h"
|
||||
#include "util/matrix.h"
|
||||
#include "util/timer.h"
|
||||
#include "util/type_conversion.h"
|
||||
|
||||
// Dispatch routines to CUBLAS and CUTLASS
|
||||
#include "cublas_dispatch.h"
|
||||
#include "cutlass_dispatch.h"
|
||||
|
||||
/******************************************************************************
|
||||
* Globals, constants and typedefs
|
||||
******************************************************************************/
|
||||
|
||||
using namespace cutlass;
|
||||
|
||||
/// CUBLAS handle
|
||||
cublasHandle_t g_cublas_handle;
|
||||
|
||||
/// The device-id of the current device
|
||||
int g_device_id = -1;
|
||||
|
||||
/// The number of timing iterations to invoke
|
||||
int g_timing_iterations = -1;
|
||||
|
||||
/// The number of randomly-sized problems to schmoo
|
||||
int g_schmoo = 0;
|
||||
|
||||
|
||||
/******************************************************************************
|
||||
* Number generation
|
||||
******************************************************************************/
|
||||
|
||||
/**
|
||||
* Simple low-integer generator
|
||||
*/
|
||||
struct simple_gen
|
||||
{
|
||||
std::default_random_engine generator;
|
||||
std::uniform_int_distribution<int> distribution;
|
||||
|
||||
/// Constructor
|
||||
simple_gen(int max) : distribution(max * -1, max)
|
||||
{}
|
||||
|
||||
/// Functor
|
||||
int operator()()
|
||||
{
|
||||
return distribution(generator);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
||||
/******************************************************************************
|
||||
* Test execution
|
||||
******************************************************************************/
|
||||
|
||||
|
||||
/**
|
||||
* Compute C = (alpha * A * B) + (beta * C)
|
||||
*/
|
||||
template <
|
||||
typename test_func_t, ///< Test function type
|
||||
matrix_transform_t::kind_t TransformA, ///< Transformation op for matrix A
|
||||
matrix_transform_t::kind_t TransformB, ///< Transformation op for matrix B
|
||||
typename value_t, ///< Multiplicand value type (matrices A and B)
|
||||
typename accum_t> ///< Accumulator value type (matrix C and scalars)
|
||||
bool test(
|
||||
int m, ///< Height of C in rows
|
||||
int n, ///< Width of C in columns
|
||||
int k, ///< Width (height) of A (B)
|
||||
accum_t alpha, ///< Multiplicand scalar
|
||||
accum_t beta) ///< Addend scalar
|
||||
{
|
||||
cudaStream_t stream = 0;
|
||||
|
||||
//
|
||||
// Initialize matrices
|
||||
//
|
||||
|
||||
matrix<value_t> A(
|
||||
(TransformA == matrix_transform_t::NonTranspose) ? m : k,
|
||||
(TransformA == matrix_transform_t::NonTranspose) ? k : m);
|
||||
|
||||
matrix<value_t> B(
|
||||
(TransformB == matrix_transform_t::NonTranspose) ? k : n,
|
||||
(TransformB == matrix_transform_t::NonTranspose) ? n : k);
|
||||
|
||||
matrix<accum_t> C(m, n);
|
||||
|
||||
// initialized matrices with small values precisely representable as integers
|
||||
simple_gen a_gen(3);
|
||||
simple_gen b_gen(5);
|
||||
A.fill_random(a_gen);
|
||||
B.fill_random(b_gen);
|
||||
C.fill_ramp(0,0);
|
||||
|
||||
// // Alternatively, initialize with procedural values to simplify debugging incorrect results
|
||||
// A.fill_ramp(1,2);
|
||||
// B.fill_ramp(1,1);
|
||||
|
||||
// Sync to device
|
||||
A.sync_device();
|
||||
B.sync_device();
|
||||
C.sync_device();
|
||||
|
||||
CUDA_PERROR(cudaPeekAtLastError());
|
||||
CUDA_PERROR(cudaDeviceSynchronize());
|
||||
|
||||
//
|
||||
// Run test once with debug-synchronous enabled and check result
|
||||
//
|
||||
|
||||
if (!g_schmoo) printf("\n");
|
||||
|
||||
test_func_t test_func;
|
||||
|
||||
C.fill_ramp(0, 0);
|
||||
C.sync_device();
|
||||
|
||||
cudaError_t error = test_func(
|
||||
g_cublas_handle,
|
||||
m,
|
||||
n,
|
||||
k,
|
||||
A.d_data(),
|
||||
B.d_data(),
|
||||
C.d_data(),
|
||||
alpha,
|
||||
beta,
|
||||
stream,
|
||||
!g_schmoo).result;
|
||||
|
||||
bool not_applicable = (error == cudaErrorInvalidValue);
|
||||
bool is_failed = false;
|
||||
if (not_applicable)
|
||||
{
|
||||
printf(", NA");
|
||||
}
|
||||
else
|
||||
{
|
||||
CUDA_PERROR(error);
|
||||
|
||||
// Compute reference check if wont take too long on CPU
|
||||
if ((!g_schmoo) && (m * n <= 1024 * 1024))
|
||||
{
|
||||
matrix<accum_t> ref_C(m, n);
|
||||
ref_C.fill_ramp(0, 0);
|
||||
ref_C.gemm(TransformA, TransformB, alpha, A, B, beta);
|
||||
C.sync_host();
|
||||
|
||||
is_failed = (C != ref_C);
|
||||
|
||||
if (!g_schmoo)
|
||||
{
|
||||
if (is_failed)
|
||||
{
|
||||
printf("FAIL, ");
|
||||
std::ofstream file_a("a.csv");
|
||||
A.write_matrix(file_a);
|
||||
std::ofstream file_b("b.csv");
|
||||
B.write_matrix(file_b);
|
||||
std::ofstream file_d("gemm-REF.csv");
|
||||
ref_C.write_matrix(file_d);
|
||||
std::ofstream file_c("gemm-GPU.csv");
|
||||
C.write_matrix(file_c);
|
||||
}
|
||||
else
|
||||
{
|
||||
printf("PASS, ");
|
||||
}
|
||||
}
|
||||
}
|
||||
fflush(stdout);
|
||||
|
||||
//
|
||||
// Warmup and timing iterations
|
||||
//
|
||||
|
||||
if (g_timing_iterations > 0)
|
||||
{
|
||||
// Warmup for 1/100 of the timing iterations (minimum of 2)
|
||||
for (int i = 0; i < __NV_STD_MAX(2, (g_timing_iterations + 99) / 100); ++i)
|
||||
{
|
||||
CUDA_PERROR(test_func(
|
||||
g_cublas_handle,
|
||||
m,
|
||||
n,
|
||||
k,
|
||||
A.d_data(),
|
||||
B.d_data(),
|
||||
C.d_data(),
|
||||
alpha,
|
||||
beta,
|
||||
stream,
|
||||
false).result);
|
||||
}
|
||||
}
|
||||
|
||||
// Conduct timing iterations
|
||||
double elapsed_ms = 0;
|
||||
gpu_timer timer;
|
||||
timer.start();
|
||||
|
||||
for (int i = 0; i < g_timing_iterations; i++)
|
||||
{
|
||||
CUDA_PERROR(test_func(
|
||||
g_cublas_handle,
|
||||
m,
|
||||
n,
|
||||
k,
|
||||
A.d_data(),
|
||||
B.d_data(),
|
||||
C.d_data(),
|
||||
alpha,
|
||||
beta,
|
||||
stream,
|
||||
false).result);
|
||||
}
|
||||
|
||||
timer.stop();
|
||||
elapsed_ms += timer.elapsed_millis();
|
||||
double avg_ms = elapsed_ms / g_timing_iterations;
|
||||
|
||||
// Display performance
|
||||
if (g_timing_iterations > 0)
|
||||
{
|
||||
int64_t num_flops = (2 * int64_t(m) * int64_t(n) * int64_t(k)) + (2 * int64_t(m) * int64_t(n));
|
||||
double gflops_per_sec = double(num_flops) / avg_ms / 1.0e6;
|
||||
|
||||
if (g_schmoo)
|
||||
{
|
||||
if (is_failed)
|
||||
printf("F");
|
||||
|
||||
printf(", %.3f", gflops_per_sec);
|
||||
|
||||
// Sleep for a few milliseconds to cool
|
||||
sleep_millis(10);
|
||||
}
|
||||
else
|
||||
{
|
||||
printf("Avg runtime: %.3f ms, total flops: %lld, GFLOP/s: %.2f\n",
|
||||
avg_ms,
|
||||
num_flops,
|
||||
gflops_per_sec);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
}
|
||||
|
||||
return is_failed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute C = (alpha * A * B) + (beta * C)
|
||||
*/
|
||||
template <
|
||||
math_operation_class_t math_op,
|
||||
matrix_transform_t::kind_t TransformA, ///< Transformation op for matrix A
|
||||
matrix_transform_t::kind_t TransformB, ///< Transformation op for matrix B
|
||||
typename value_t, ///< Multiplicand value type (matrices A and B)
|
||||
typename accum_t> ///< Accumulator value type (matrix C and scalars)
|
||||
bool test(
|
||||
int m, ///< Height of C in rows
|
||||
int n, ///< Width of C in columns
|
||||
int k, ///< Width (height) of A (B)
|
||||
accum_t alpha, ///< Multiplicand scalar
|
||||
accum_t beta) ///< Addend scalar
|
||||
{
|
||||
uint64_t flop_base = 1ull << 41;
|
||||
int max_timing_iterations = 10000;
|
||||
int min_timing_iterations = 10;
|
||||
|
||||
bool test_error = false;
|
||||
|
||||
// Scale the number of timing iterations with respect to problem size (if not specified on commandline)
|
||||
if ((g_timing_iterations < 0) || g_schmoo)
|
||||
{
|
||||
uint64_t num_flops = (2 * uint64_t(m) * uint64_t(n) * uint64_t(k)) + (2 * uint64_t(m) * uint64_t(n));
|
||||
g_timing_iterations = (int) ((flop_base / sizeof(value_t)) / num_flops);
|
||||
|
||||
g_timing_iterations = (int) __NV_STD_MIN(max_timing_iterations, g_timing_iterations);
|
||||
g_timing_iterations = (int) __NV_STD_MAX(min_timing_iterations, g_timing_iterations);
|
||||
}
|
||||
|
||||
if (g_schmoo)
|
||||
{
|
||||
printf("%d, %d, %d, %c%c, %d, %d",
|
||||
m, n, k,
|
||||
(TransformA == matrix_transform_t::NonTranspose) ? 'n' : 't',
|
||||
(TransformB == matrix_transform_t::NonTranspose) ? 'n' : 't',
|
||||
m * n,
|
||||
g_timing_iterations);
|
||||
}
|
||||
else
|
||||
{
|
||||
printf("\n------------------------------------------------------------\n");
|
||||
printf("%dx%dx%d, GEMM_%c%c, %d C elements, %d timing iterations\n",
|
||||
m, n, k,
|
||||
(TransformA == matrix_transform_t::NonTranspose) ? 'n' : 't',
|
||||
(TransformB == matrix_transform_t::NonTranspose) ? 'n' : 't',
|
||||
m * n,
|
||||
g_timing_iterations);
|
||||
}
|
||||
fflush(stdout);
|
||||
|
||||
// CUBLAS
|
||||
test_error |= test<
|
||||
cublas_gemm<gemm::tiling_strategy::Unknown, math_op, TransformA, TransformB, value_t, accum_t>,
|
||||
TransformA,
|
||||
TransformB,
|
||||
value_t,
|
||||
accum_t>(m, n, k, accum_t(alpha), accum_t(beta));
|
||||
|
||||
// CUTLASS
|
||||
test_error |= test<
|
||||
cutlass_gemm_dispatch<gemm::tiling_strategy::Small, math_op, TransformA, TransformB, value_t, accum_t>,
|
||||
TransformA,
|
||||
TransformB,
|
||||
value_t,
|
||||
accum_t>(m, n, k, accum_t(alpha), accum_t(beta));
|
||||
|
||||
test_error |= test<
|
||||
cutlass_gemm_dispatch<gemm::tiling_strategy::Medium, math_op, TransformA, TransformB, value_t, accum_t>,
|
||||
TransformA,
|
||||
TransformB,
|
||||
value_t,
|
||||
accum_t>(m, n, k, accum_t(alpha), accum_t(beta));
|
||||
|
||||
test_error |= test<
|
||||
cutlass_gemm_dispatch<gemm::tiling_strategy::Large, math_op, TransformA, TransformB, value_t, accum_t>,
|
||||
TransformA,
|
||||
TransformB,
|
||||
value_t,
|
||||
accum_t>(m, n, k, accum_t(alpha), accum_t(beta));
|
||||
|
||||
test_error |= test<
|
||||
cutlass_gemm_dispatch<gemm::tiling_strategy::Tall, math_op, TransformA, TransformB, value_t, accum_t>,
|
||||
TransformA,
|
||||
TransformB,
|
||||
value_t,
|
||||
accum_t>(m, n, k, accum_t(alpha), accum_t(beta));
|
||||
|
||||
test_error |= test<
|
||||
cutlass_gemm_dispatch<gemm::tiling_strategy::Wide, math_op, TransformA, TransformB, value_t, accum_t>,
|
||||
TransformA,
|
||||
TransformB,
|
||||
value_t,
|
||||
accum_t>(m, n, k, accum_t(alpha), accum_t(beta));
|
||||
|
||||
test_error |= test<
|
||||
cutlass_gemm_dispatch<gemm::tiling_strategy::Huge, math_op, TransformA, TransformB, value_t, accum_t>,
|
||||
TransformA,
|
||||
TransformB,
|
||||
value_t,
|
||||
accum_t>(m, n, k, accum_t(alpha), accum_t(beta));
|
||||
|
||||
return test_error;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/******************************************************************************
|
||||
* Main
|
||||
******************************************************************************/
|
||||
|
||||
|
||||
/**
|
||||
* Main
|
||||
*/
|
||||
int main(int argc, const char **argv)
|
||||
{
|
||||
//
|
||||
// Problem type (compiler-supplied so we don't compile everything)
|
||||
//
|
||||
|
||||
// Define value_t and accum_t (multiplicand and accumulator types, respectively)
|
||||
#if defined(TEST_SGEMM)
|
||||
typedef float value_t;
|
||||
typedef float accum_t;
|
||||
const math_operation_class_t math_op = math_operation_class_t::scalar;
|
||||
#elif defined(TEST_DGEMM)
|
||||
typedef double value_t;
|
||||
typedef double accum_t;
|
||||
const math_operation_class_t math_op = math_operation_class_t::scalar;
|
||||
#elif defined(TEST_HGEMM)
|
||||
typedef __half value_t;
|
||||
typedef __half accum_t;
|
||||
const math_operation_class_t math_op = math_operation_class_t::scalar;
|
||||
#elif defined(TEST_IGEMM)
|
||||
typedef int8_t value_t;
|
||||
typedef int32_t accum_t;
|
||||
const math_operation_class_t math_op = math_operation_class_t::scalar;
|
||||
#elif defined(TEST_WGEMM)
|
||||
typedef half value_t;
|
||||
typedef float accum_t;
|
||||
const math_operation_class_t math_op = math_operation_class_t::matrix;
|
||||
#else
|
||||
#error Unknown GEMM type requested.
|
||||
#endif
|
||||
|
||||
|
||||
// Define transpose constants
|
||||
#ifdef TRANSPOSE_A
|
||||
static const matrix_transform_t::kind_t TransformA = matrix_transform_t::Transpose;
|
||||
#else
|
||||
static const matrix_transform_t::kind_t TransformA = matrix_transform_t::NonTranspose;
|
||||
#endif
|
||||
|
||||
#ifdef TRANSPOSE_B
|
||||
static const matrix_transform_t::kind_t TransformB = matrix_transform_t::Transpose;
|
||||
#else
|
||||
static const matrix_transform_t::kind_t TransformB = matrix_transform_t::NonTranspose;
|
||||
#endif
|
||||
|
||||
|
||||
//
|
||||
// Commandline parsing
|
||||
//
|
||||
|
||||
// Initialize command line
|
||||
command_line args(argc, argv);
|
||||
|
||||
int m_factor = args.device_prop.multiProcessorCount * 128;
|
||||
int m = round_nearest(4096, m_factor);
|
||||
int k = 4096;
|
||||
int n = 4096;
|
||||
float alpha = 1.0;
|
||||
float beta = 0.0;
|
||||
|
||||
g_device_id = args.device_id;
|
||||
args.get_cmd_line_argument("m", m);
|
||||
args.get_cmd_line_argument("n", n);
|
||||
args.get_cmd_line_argument("k", k);
|
||||
args.get_cmd_line_argument("i", g_timing_iterations);
|
||||
args.get_cmd_line_argument("alpha", alpha);
|
||||
args.get_cmd_line_argument("beta", beta);
|
||||
args.get_cmd_line_argument("schmoo", g_schmoo);
|
||||
|
||||
// Print usage
|
||||
if (args.check_cmd_line_flag("help"))
|
||||
{
|
||||
printf("%s "
|
||||
"[--help] "
|
||||
"[--i=<timing iterations>] "
|
||||
"[--device=<device-id>] "
|
||||
"[--alpha=<alpha> --beta=<beta>] "
|
||||
"[--schmoo=<samples> || --m=<height> --n=<width> --k=<depth>]"
|
||||
"\n", argv[0]);
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// Initialize cuBLAS
|
||||
if (cublasCreate(&g_cublas_handle) != CUBLAS_STATUS_SUCCESS)
|
||||
{
|
||||
fprintf(stderr, "cublasCreate() failed\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
bool test_error = false;
|
||||
|
||||
if (g_schmoo)
|
||||
{
|
||||
// Run a schmoo of problem sizes
|
||||
printf("M, N, K, transpose, total_flops, timing_iterations, sol_flop/s, cublas_sol, cutlass_small_sol, cutlass_med_sol, cutlass_large_sol, cutlass_tall_sol, cutlass_wide_sol, cutlass_huge_sol\n");
|
||||
|
||||
// Generate power-law distribution from [32, 16384)
|
||||
std::mt19937 gen(0);
|
||||
std::uniform_real_distribution<float> dis(5, 14);
|
||||
for (int i = 0; i < g_schmoo; ++i)
|
||||
{
|
||||
int m = int(pow(float(2), dis(gen)));
|
||||
int n = int(pow(float(2), dis(gen)));
|
||||
int k = int(pow(float(2), dis(gen)));
|
||||
|
||||
// Round m and n to nearest multiple of 32 if < 128, otherwise to the nearest 128
|
||||
m = (m < 128) ?
|
||||
round_nearest(m, 32) :
|
||||
round_nearest(m, 128);
|
||||
n = (n < 128) ?
|
||||
round_nearest(n, 32) :
|
||||
round_nearest(n, 128);
|
||||
|
||||
// Round k to the nearest 16
|
||||
k = (sizeof(value_t) == 1) ?
|
||||
round_nearest(k, 32) :
|
||||
round_nearest(k, 16);
|
||||
|
||||
test_error |= test<math_op, TransformA, TransformB, value_t, accum_t>(
|
||||
m, n, k,
|
||||
from_float<accum_t>(alpha),
|
||||
from_float<accum_t>(beta));
|
||||
|
||||
printf("\n"); fflush(stdout);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Test a single GEMM problem size
|
||||
test_error |= test<math_op, TransformA, TransformB, value_t, accum_t>(
|
||||
m,
|
||||
n,
|
||||
k,
|
||||
from_float<accum_t>(alpha),
|
||||
from_float<accum_t>(beta));
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
cublasDestroy(g_cublas_handle);
|
||||
|
||||
return test_error;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
/******************************************************************************
|
||||
* Copyright (c) 2011-2017, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are not permitted.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
******************************************************************************/
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* \file
|
||||
* Utility for parsing command line arguments
|
||||
*/
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <sstream>
|
||||
#include <iostream>
|
||||
#include <limits>
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <cutlass/util/debug.h>
|
||||
|
||||
|
||||
namespace cutlass {
|
||||
|
||||
/******************************************************************************
|
||||
* command_line
|
||||
******************************************************************************/
|
||||
|
||||
/**
|
||||
* Utility for parsing command line arguments
|
||||
*/
|
||||
struct command_line
|
||||
{
|
||||
|
||||
std::vector<std::string> keys;
|
||||
std::vector<std::string> values;
|
||||
std::vector<std::string> args;
|
||||
int device_id;
|
||||
cudaDeviceProp device_prop;
|
||||
float device_giga_bandwidth;
|
||||
size_t device_free_physmem;
|
||||
size_t device_total_physmem;
|
||||
|
||||
/**
|
||||
* Constructor
|
||||
*/
|
||||
command_line(int argc, const char **argv, int device_id = -1) :
|
||||
keys(10),
|
||||
values(10),
|
||||
device_id(device_id)
|
||||
{
|
||||
using namespace std;
|
||||
|
||||
for (int i = 1; i < argc; i++)
|
||||
{
|
||||
string arg = argv[i];
|
||||
|
||||
if ((arg[0] != '-') || (arg[1] != '-'))
|
||||
{
|
||||
args.push_back(arg);
|
||||
continue;
|
||||
}
|
||||
|
||||
string::size_type pos;
|
||||
string key, val;
|
||||
if ((pos = arg.find('=')) == string::npos) {
|
||||
key = string(arg, 2, arg.length() - 2);
|
||||
val = "";
|
||||
} else {
|
||||
key = string(arg, 2, pos - 2);
|
||||
val = string(arg, pos + 1, arg.length() - 1);
|
||||
}
|
||||
|
||||
keys.push_back(key);
|
||||
values.push_back(val);
|
||||
}
|
||||
|
||||
// Initialize device
|
||||
CUDA_PERROR_EXIT(device_init());
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Checks whether a flag "--<flag>" is present in the commandline
|
||||
*/
|
||||
bool check_cmd_line_flag(const char* arg_name)
|
||||
{
|
||||
using namespace std;
|
||||
|
||||
for (int i = 0; i < int(keys.size()); ++i)
|
||||
{
|
||||
if (keys[i] == string(arg_name))
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns number of naked (non-flag and non-key-value) commandline parameters
|
||||
*/
|
||||
template <typename value_t>
|
||||
int num_naked_args()
|
||||
{
|
||||
return args.size();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the commandline parameter for a given index (not including flags)
|
||||
*/
|
||||
template <typename value_t>
|
||||
void get_cmd_line_argument(int index, value_t &val)
|
||||
{
|
||||
using namespace std;
|
||||
if (index < args.size()) {
|
||||
istringstream str_stream(args[index]);
|
||||
str_stream >> val;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the value specified for a given commandline parameter --<flag>=<value>
|
||||
*/
|
||||
template <typename value_t>
|
||||
void get_cmd_line_argument(const char *arg_name, value_t &val)
|
||||
{
|
||||
using namespace std;
|
||||
|
||||
for (int i = 0; i < int(keys.size()); ++i)
|
||||
{
|
||||
if (keys[i] == string(arg_name))
|
||||
{
|
||||
istringstream str_stream(values[i]);
|
||||
str_stream >> val;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the values specified for a given commandline parameter --<flag>=<value>,<value>*
|
||||
*/
|
||||
template <typename value_t>
|
||||
void get_cmd_line_arguments(
|
||||
const char *arg_name,
|
||||
std::vector<value_t> &vals,
|
||||
char sep = ',')
|
||||
{
|
||||
using namespace std;
|
||||
|
||||
if (check_cmd_line_flag(arg_name))
|
||||
{
|
||||
// Clear any default values
|
||||
vals.clear();
|
||||
|
||||
// Recover from multi-value string
|
||||
for (int i = 0; i < keys.size(); ++i)
|
||||
{
|
||||
if (keys[i] == string(arg_name))
|
||||
{
|
||||
string val_string(values[i]);
|
||||
istringstream str_stream(val_string);
|
||||
string::size_type old_pos = 0;
|
||||
string::size_type new_pos = 0;
|
||||
|
||||
// Iterate <sep>-delimited values
|
||||
value_t val;
|
||||
while ((new_pos = val_string.find(sep, old_pos)) != string::npos)
|
||||
{
|
||||
if (new_pos != old_pos)
|
||||
{
|
||||
str_stream.width(new_pos - old_pos);
|
||||
str_stream >> val;
|
||||
vals.push_back(val);
|
||||
}
|
||||
|
||||
// skip over delimiter
|
||||
str_stream.ignore(1);
|
||||
old_pos = new_pos + 1;
|
||||
}
|
||||
|
||||
// Read last value
|
||||
str_stream >> val;
|
||||
vals.push_back(val);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* The number of pairs parsed
|
||||
*/
|
||||
int parsed_argc()
|
||||
{
|
||||
return (int) keys.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize device
|
||||
*/
|
||||
cudaError_t device_init()
|
||||
{
|
||||
cudaError_t error = cudaSuccess;
|
||||
|
||||
do
|
||||
{
|
||||
int deviceCount;
|
||||
if (CUDA_PERROR(error = cudaGetDeviceCount(&deviceCount))) break;
|
||||
|
||||
if (deviceCount == 0) {
|
||||
fprintf(stderr, "No devices supporting CUDA.\n");
|
||||
exit(1);
|
||||
}
|
||||
if (device_id < 0)
|
||||
{
|
||||
get_cmd_line_argument("device", device_id);
|
||||
}
|
||||
if ((device_id > deviceCount - 1) || (device_id < 0))
|
||||
{
|
||||
device_id = 0;
|
||||
}
|
||||
|
||||
if (CUDA_PERROR(error = cudaSetDevice(device_id))) break;
|
||||
|
||||
if (CUDA_PERROR(error = cudaMemGetInfo(&device_free_physmem, &device_total_physmem))) break;
|
||||
|
||||
if (CUDA_PERROR(error = cudaGetDeviceProperties(&device_prop, device_id))) break;
|
||||
|
||||
if (device_prop.major < 1) {
|
||||
fprintf(stderr, "Device does not support CUDA.\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
device_giga_bandwidth = float(device_prop.memoryBusWidth) * device_prop.memoryClockRate * 2 / 8 / 1000 / 1000;
|
||||
|
||||
} while (0);
|
||||
|
||||
return error;
|
||||
}
|
||||
|
||||
|
||||
//-------------------------------------------------------------------------
|
||||
// Utility functions
|
||||
//-------------------------------------------------------------------------
|
||||
|
||||
/// Tokenizes a comma-delimited list of string pairs delimited by ':'
|
||||
static void tokenize(
|
||||
std::vector<std::pair<std::string, std::string> > &tokens,
|
||||
std::string const &str,
|
||||
char delim = ',',
|
||||
char sep = ':')
|
||||
{
|
||||
// Home-built to avoid Boost dependency
|
||||
size_t s_idx = 0;
|
||||
size_t d_idx = std::string::npos;
|
||||
while (s_idx < str.size())
|
||||
{
|
||||
d_idx = str.find_first_of(delim, s_idx);
|
||||
|
||||
size_t end_idx = (d_idx != std::string::npos ? d_idx : str.size());
|
||||
size_t sep_idx = str.find_first_of(sep, s_idx);
|
||||
size_t offset = 1;
|
||||
if (sep_idx == std::string::npos || sep_idx >= end_idx)
|
||||
{
|
||||
sep_idx = end_idx;
|
||||
offset = 0;
|
||||
}
|
||||
|
||||
std::pair<std::string, std::string> item(
|
||||
str.substr(s_idx, sep_idx - s_idx),
|
||||
str.substr(sep_idx + offset, end_idx - sep_idx - offset));
|
||||
|
||||
tokens.push_back(item);
|
||||
s_idx = end_idx + 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// Tokenizes a comma-delimited list of string pairs delimited by ':'
|
||||
static void tokenize(
|
||||
std::vector<std::string > &tokens,
|
||||
std::string const &str,
|
||||
char delim = ',',
|
||||
char sep = ':')
|
||||
{
|
||||
std::vector<std::pair<std::string, std::string> > token_pairs;
|
||||
tokenize(token_pairs, str, delim, sep);
|
||||
for (auto const &tok : token_pairs)
|
||||
{
|
||||
tokens.push_back(tok.first);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
} // namespace cutlass
|
||||
@@ -0,0 +1,83 @@
|
||||
/******************************************************************************
|
||||
* Copyright (c) 2011-2017, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are not permitted.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
******************************************************************************/
|
||||
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* \file
|
||||
* \brief C++ exception semantics for CUDA error codes
|
||||
*/
|
||||
|
||||
#include <iosfwd>
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
|
||||
namespace cutlass {
|
||||
|
||||
|
||||
/// C++ exception wrapper for CUDA \p cudaError_t
|
||||
class cuda_exception : public std::exception
|
||||
{
|
||||
public:
|
||||
|
||||
/// Constructor
|
||||
cuda_exception(
|
||||
const char *msg = "",
|
||||
cudaError_t err = cudaErrorUnknown)
|
||||
:
|
||||
msg(msg), err(err)
|
||||
{}
|
||||
|
||||
/// Returns the explanatory string
|
||||
const char *what() const noexcept
|
||||
{
|
||||
return msg;
|
||||
}
|
||||
|
||||
/// Returns the underlying CUDA \p cudaError_t
|
||||
cudaError_t cudaError() const
|
||||
{
|
||||
return err;
|
||||
}
|
||||
|
||||
|
||||
protected:
|
||||
|
||||
/// Explanatory string
|
||||
const char *msg;
|
||||
|
||||
/// Underlying CUDA \p cudaError_t
|
||||
cudaError_t err;
|
||||
};
|
||||
|
||||
|
||||
/// Writes a cudaError_t to an output stream
|
||||
inline std::ostream & operator<<(std::ostream &out, cudaError_t result)
|
||||
{
|
||||
return out << cudaGetErrorString(result);
|
||||
}
|
||||
|
||||
/// Writes a cuda_exception instance to an output stream
|
||||
inline std::ostream & operator<<(std::ostream &out, cuda_exception const &e)
|
||||
{
|
||||
return out << e.what() << ": " << e.cudaError();
|
||||
}
|
||||
|
||||
|
||||
} // namespace cutlass
|
||||
@@ -0,0 +1,224 @@
|
||||
/******************************************************************************
|
||||
* Copyright (c) 2011-2017, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are not permitted.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
******************************************************************************/
|
||||
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* \file
|
||||
* Utilities for interacting with the opaque CUDA __half type
|
||||
*/
|
||||
|
||||
#include <stdint.h>
|
||||
#include <cuda_fp16.h>
|
||||
#include <iosfwd>
|
||||
|
||||
namespace cutlass {
|
||||
|
||||
|
||||
/******************************************************************************
|
||||
* half_t
|
||||
******************************************************************************/
|
||||
|
||||
/**
|
||||
* Host-based fp16 data type compatible and convertible with __half
|
||||
*/
|
||||
struct half_t
|
||||
{
|
||||
uint16_t __x;
|
||||
|
||||
/// Constructor from __half
|
||||
half_t(const __half &other)
|
||||
{
|
||||
__x = reinterpret_cast<const uint16_t&>(other);
|
||||
}
|
||||
|
||||
/// Constructor from integer
|
||||
half_t(int a)
|
||||
{
|
||||
*this = half_t(float(a));
|
||||
}
|
||||
|
||||
|
||||
/// Constructor from float
|
||||
half_t(float a)
|
||||
{
|
||||
uint32_t ia = *reinterpret_cast<uint32_t*>(&a);
|
||||
uint16_t ir;
|
||||
|
||||
ir = (ia >> 16) & 0x8000;
|
||||
|
||||
if ((ia & 0x7f800000) == 0x7f800000)
|
||||
{
|
||||
if ((ia & 0x7fffffff) == 0x7f800000)
|
||||
{
|
||||
ir |= 0x7c00; /* infinity */
|
||||
}
|
||||
else
|
||||
{
|
||||
ir = 0x7fff; /* canonical NaN */
|
||||
}
|
||||
}
|
||||
else if ((ia & 0x7f800000) >= 0x33000000)
|
||||
{
|
||||
int32_t shift = (int32_t) ((ia >> 23) & 0xff) - 127;
|
||||
if (shift > 15)
|
||||
{
|
||||
ir |= 0x7c00; /* infinity */
|
||||
}
|
||||
else
|
||||
{
|
||||
ia = (ia & 0x007fffff) | 0x00800000; /* extract mantissa */
|
||||
if (shift < -14)
|
||||
{ /* denormal */
|
||||
ir |= ia >> (-1 - shift);
|
||||
ia = ia << (32 - (-1 - shift));
|
||||
}
|
||||
else
|
||||
{ /* normal */
|
||||
ir |= ia >> (24 - 11);
|
||||
ia = ia << (32 - (24 - 11));
|
||||
ir = ir + ((14 + shift) << 10);
|
||||
}
|
||||
/* IEEE-754 round to nearest of even */
|
||||
if ((ia > 0x80000000) || ((ia == 0x80000000) && (ir & 1)))
|
||||
{
|
||||
ir++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this->__x = ir;
|
||||
}
|
||||
|
||||
/// Cast to __half
|
||||
operator __half() const
|
||||
{
|
||||
return reinterpret_cast<const __half&>(__x);
|
||||
}
|
||||
|
||||
/// Cast to float
|
||||
operator float() const
|
||||
{
|
||||
int sign = ((this->__x >> 15) & 1);
|
||||
int exp = ((this->__x >> 10) & 0x1f);
|
||||
int mantissa = (this->__x & 0x3ff);
|
||||
uint32_t f = 0;
|
||||
|
||||
if (exp > 0 && exp < 31)
|
||||
{
|
||||
// normal
|
||||
exp += 112;
|
||||
f = (sign << 31) | (exp << 23) | (mantissa << 13);
|
||||
}
|
||||
else if (exp == 0)
|
||||
{
|
||||
if (mantissa)
|
||||
{
|
||||
// subnormal
|
||||
exp += 113;
|
||||
while ((mantissa & (1 << 10)) == 0)
|
||||
{
|
||||
mantissa <<= 1;
|
||||
exp--;
|
||||
}
|
||||
mantissa &= 0x3ff;
|
||||
f = (sign << 31) | (exp << 23) | (mantissa << 13);
|
||||
}
|
||||
else
|
||||
{
|
||||
// zero
|
||||
f = 0;
|
||||
}
|
||||
}
|
||||
else if (exp == 31)
|
||||
{
|
||||
if (mantissa)
|
||||
{
|
||||
f = 0x7fffffff; // not a number
|
||||
}
|
||||
else
|
||||
{
|
||||
f = (0xff << 23) | (sign << 31); // inf
|
||||
}
|
||||
}
|
||||
return *reinterpret_cast<float const *>(&f);
|
||||
}
|
||||
|
||||
|
||||
/// Get raw storage
|
||||
uint16_t raw()
|
||||
{
|
||||
return this->__x;
|
||||
}
|
||||
|
||||
/// Assignment by sum
|
||||
bool operator ==(const half_t &other)
|
||||
{
|
||||
return (this->__x == other.__x);
|
||||
}
|
||||
|
||||
/// Increment
|
||||
half_t& operator +=(const half_t &rhs)
|
||||
{
|
||||
*this = half_t(float(*this) + float(rhs));
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Decrement
|
||||
half_t& operator -=(const half_t &rhs)
|
||||
{
|
||||
*this = half_t(float(*this) - float(rhs));
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Multiply
|
||||
half_t operator*(const half_t &other)
|
||||
{
|
||||
return half_t(float(*this) * float(other));
|
||||
}
|
||||
|
||||
/// Multiply
|
||||
half_t operator+(const half_t &other)
|
||||
{
|
||||
return half_t(float(*this) + float(other));
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
|
||||
/******************************************************************************
|
||||
* I/O stream overloads
|
||||
******************************************************************************/
|
||||
|
||||
/// Insert formatted \p half_t into the output stream
|
||||
std::ostream& operator<<(std::ostream &out, const half_t &x)
|
||||
{
|
||||
out << (float)x;
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
/// Insert formatted \p __half into the output stream
|
||||
std::ostream& operator<<(std::ostream &out, const __half &x)
|
||||
{
|
||||
return out << half_t(x);
|
||||
}
|
||||
|
||||
|
||||
} // namespace cutlass
|
||||
@@ -0,0 +1,495 @@
|
||||
/******************************************************************************
|
||||
* Copyright (c) 2011-2017, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are not permitted.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
******************************************************************************/
|
||||
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* \file
|
||||
* Matrix data structure providing basic CPU-based algorithms and
|
||||
* operations that can be cloned and synchronized in GPU device memory
|
||||
*/
|
||||
|
||||
#include <vector>
|
||||
#include <fstream>
|
||||
|
||||
#include <cutlass/util/debug.h>
|
||||
#include "../cutlass/util/matrix_transform.h"
|
||||
#include "half.h"
|
||||
|
||||
|
||||
namespace cutlass {
|
||||
|
||||
/**
|
||||
* \brief Matrix data structure providing basic CPU-based algorithms and
|
||||
* operations that be synchronized with a GPU-based replica
|
||||
*/
|
||||
template <typename value_t>
|
||||
struct matrix
|
||||
{
|
||||
// Host value type (must be convertible to/from value_t)
|
||||
typedef typename nv_std::conditional<
|
||||
(nv_std::is_same<value_t, __half>::value), // If (value_t == __half) ...
|
||||
half_t, // ... use half_t internally for host storage, else...
|
||||
value_t>::type // ... use value_t directly
|
||||
host_value_t;
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// Data members
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
private:
|
||||
|
||||
/// M dimension (height in rows)
|
||||
int _m;
|
||||
|
||||
/// N dimension (width in columns)
|
||||
int _n;
|
||||
|
||||
/// Data array on host
|
||||
std::vector<host_value_t> _h_data;
|
||||
|
||||
/// Clone of data array on GPU device
|
||||
value_t *_d_data;
|
||||
|
||||
/// GPU Device identifier that clone synchronizes with
|
||||
int _device_id;
|
||||
|
||||
public:
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// Lifetime and synchronization
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Constructor: zero-initializes the matrix.
|
||||
*/
|
||||
matrix(
|
||||
int m, ///< Height of the matrix in rows
|
||||
int n) ///< Width of the matrix in columns
|
||||
:
|
||||
_m(m),
|
||||
_n(n),
|
||||
_d_data(NULL),
|
||||
_device_id(0)
|
||||
{
|
||||
_h_data.resize(_m * _n, 0);
|
||||
CUDA_PERROR_EXIT(cudaMalloc((void ** )&_d_data, sizeof(value_t) * _m * _n));
|
||||
CUDA_PERROR_EXIT(cudaGetDevice(&_device_id));
|
||||
}
|
||||
|
||||
/// Destructor
|
||||
~matrix()
|
||||
{
|
||||
if (_d_data)
|
||||
{
|
||||
CUDA_PERROR_EXIT(cudaFree(_d_data));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Synchronize the GPU-based replica with the current host-based matrix data
|
||||
*/
|
||||
void sync_device()
|
||||
{
|
||||
size_t bytes = _m * _n * sizeof(value_t);
|
||||
CUDA_PERROR_EXIT(cudaMemcpy(_d_data, &_h_data[0], bytes, cudaMemcpyHostToDevice));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Synchronize the host-based replica with the current GPU-based matrix data
|
||||
*/
|
||||
void sync_host()
|
||||
{
|
||||
size_t bytes = _m * _n * sizeof(value_t);
|
||||
CUDA_PERROR_EXIT(cudaMemcpy(&_h_data[0], _d_data, bytes, cudaMemcpyDeviceToHost));
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// Inspectors
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Return the height of the matrix, subject to the optional \p transpose_op
|
||||
*/
|
||||
int height(matrix_transform_t transpose_op = matrix_transform_t::NonTranspose) const
|
||||
{
|
||||
switch (transpose_op)
|
||||
{
|
||||
case matrix_transform_t::NonTranspose : return _m;
|
||||
case matrix_transform_t::Transpose : return _n;
|
||||
default: return -1;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Return the width of the matrix, subject to the optional \p transpose_op
|
||||
*/
|
||||
int width(matrix_transform_t transpose_op = matrix_transform_t::NonTranspose) const
|
||||
{
|
||||
switch (transpose_op)
|
||||
{
|
||||
case matrix_transform_t::NonTranspose : return _n;
|
||||
case matrix_transform_t::Transpose : return _m;
|
||||
default: return -1;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Return item at (x, y) coordinate of matrix, subject to the optional \p transform op
|
||||
*/
|
||||
host_value_t get(
|
||||
int x,
|
||||
int y,
|
||||
matrix_transform_t transpose_op = matrix_transform_t::NonTranspose) const
|
||||
{
|
||||
switch (transpose_op)
|
||||
{
|
||||
case matrix_transform_t::NonTranspose : return _h_data[y + (x * _m)];
|
||||
case matrix_transform_t::Transpose : return _h_data[x + (y * _m)];
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Return the distance (in items) within memory between elements of two
|
||||
* consecutive columns which have the same row index, subject to the optional \p transform op
|
||||
*/
|
||||
int leading_dim(matrix_transform_t transpose_op = matrix_transform_t::NonTranspose) const
|
||||
{
|
||||
switch (transpose_op)
|
||||
{
|
||||
case matrix_transform_t::NonTranspose : return _m;
|
||||
case matrix_transform_t::Transpose : return _n;
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get host data pointer
|
||||
*/
|
||||
value_t* h_data()
|
||||
{
|
||||
return _h_data.data();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get host data pointer
|
||||
*/
|
||||
value_t const* h_data() const
|
||||
{
|
||||
return _h_data.data();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get device data pointer
|
||||
*/
|
||||
value_t const* d_data() const
|
||||
{
|
||||
return _d_data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get device data pointer
|
||||
*/
|
||||
value_t * d_data()
|
||||
{
|
||||
return _d_data;
|
||||
}
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// Initialization
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Initialize matrix values with a 2D "ramp" defined as
|
||||
* <tt>values(x, y) = (y * rs) + (x * cs)</tt>
|
||||
*/
|
||||
void fill_ramp(
|
||||
host_value_t rs,
|
||||
host_value_t cs)
|
||||
{
|
||||
for (int x = 0; x < _n; x++)
|
||||
{
|
||||
for (int y = 0; y < _m; y++)
|
||||
{
|
||||
_h_data[y + (x * _m)] = host_value_t((y * rs) + (x * cs));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Initialize matrix values such that all the elements of the principal diagonal
|
||||
* are ones and all other elements are zeros
|
||||
*/
|
||||
void fill_identity()
|
||||
{
|
||||
for (int j = 0; j < _n; j++)
|
||||
{
|
||||
for (int i = 0; i < _m; i++)
|
||||
{
|
||||
_h_data[i + j * _m] = host_value_t(i == j ? 1 : 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Initialize matrix values using the random number \p generator. The
|
||||
* \p generator reference is assumed to be a nullary functor that returns
|
||||
* values convertible to the matrix \p value_t.
|
||||
*/
|
||||
template <typename T>
|
||||
void fill_random(T & generator)
|
||||
{
|
||||
for (int j = 0; j < _n; j++)
|
||||
{
|
||||
for (int i = 0; i < _m; i++)
|
||||
{
|
||||
_h_data[i + j * _m] = (value_t) generator();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Element-wise matrix addition
|
||||
*/
|
||||
matrix & operator+=(matrix const &mat)
|
||||
{
|
||||
for (int j = 0; j < _n; j++)
|
||||
{
|
||||
for (int i = 0; i < _m; i++)
|
||||
{
|
||||
_h_data[i + j * _m] += mat._h_data[i + j * _m];
|
||||
}
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Element-wise matrix subtraction
|
||||
*/
|
||||
matrix & operator-=(matrix const &mat)
|
||||
{
|
||||
for (int j = 0; j < _n; j++)
|
||||
{
|
||||
for (int i = 0; i < _m; i++)
|
||||
{
|
||||
_h_data[i + j * _m] -= mat._h_data[i + j * _m];
|
||||
}
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// Output
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Prints matrix in CSV to output stream
|
||||
*/
|
||||
template <typename _hv_t>
|
||||
std::ostream & write_matrix(std::ostream &out, _hv_t)
|
||||
{
|
||||
for (int i = 0; i < _m; i++)
|
||||
{
|
||||
for (int j = 0; j < _n; j++)
|
||||
{
|
||||
out << (j ? "," : "") << _h_data[i + j * _m];
|
||||
}
|
||||
out << "\n";
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Prints matrix in CSV to output stream
|
||||
*/
|
||||
std::ostream & write_matrix(std::ostream &out, int8_t)
|
||||
{
|
||||
for (int i = 0; i < _m; i++)
|
||||
{
|
||||
for (int j = 0; j < _n; j++)
|
||||
{
|
||||
out << (j ? "," : "") << int32_t(_h_data[i + j * _m]);
|
||||
}
|
||||
out << "\n";
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Prints matrix in CSV to output stream
|
||||
*/
|
||||
std::ostream & write_matrix(std::ostream &out)
|
||||
{
|
||||
return write_matrix(out, _h_data[0]);
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// Floating point "almost-equal" utilities
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
static bool almost_equal_ulps(half_t a, half_t b, int max_ulps)
|
||||
{
|
||||
if (a == b)
|
||||
return true;
|
||||
|
||||
int32_t int_diff = abs(a.raw() - b.raw());
|
||||
if (int_diff <= max_ulps)
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
static bool almost_equal_ulps(float a, float b, int max_ulps)
|
||||
{
|
||||
if (a == b)
|
||||
return true;
|
||||
int32_t int_diff = abs(*(int32_t*)&a - *(int32_t*)&b);
|
||||
if (int_diff <= max_ulps)
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
static bool almost_equal_ulps(double a, double b, int max_ulps)
|
||||
{
|
||||
if (a == b)
|
||||
return true;
|
||||
int64_t int_diff = abs(*(int64_t*)&a - *(int64_t*)&b);
|
||||
if (int_diff <= max_ulps)
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool almost_equal_ulps(int32_t a, int32_t b, int max_ulps)
|
||||
{
|
||||
return (a == b);
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// matrix operations
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
|
||||
/**
|
||||
* Returns matrix equality
|
||||
*/
|
||||
bool operator==(const matrix<value_t> &mat) const
|
||||
{
|
||||
int max_ulps = 30;
|
||||
|
||||
if (_m != mat._m || _n != mat._n)
|
||||
{
|
||||
fprintf(stderr, "Error: dimension mismatch during matrix comparison.\n"); exit(1);
|
||||
}
|
||||
|
||||
for (int j = 0; j < _n; j++)
|
||||
{
|
||||
for (int i = 0; i < _m; i++)
|
||||
{
|
||||
if (!almost_equal_ulps(_h_data[i + j * _m], mat._h_data[i + j * _m], max_ulps))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns matrix inequality
|
||||
*/
|
||||
bool operator!=(const matrix<value_t> &mat) const
|
||||
{
|
||||
return !(*this == mat);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Computes this = (alpha * op(A) * op(B)) + (beta * this), specialized for gemm_nn
|
||||
*/
|
||||
template <typename multiplicand_t>
|
||||
void gemm(
|
||||
matrix_transform_t transform_a,
|
||||
matrix_transform_t transform_b,
|
||||
host_value_t alpha,
|
||||
const matrix<multiplicand_t> &A,
|
||||
const matrix<multiplicand_t> &B,
|
||||
host_value_t beta)
|
||||
{
|
||||
// Sanity check dimensions
|
||||
if ((_m != A.height(transform_a)) ||
|
||||
(_n != B.width(transform_b)) ||
|
||||
(A.width(transform_a) != B.height(transform_b)))
|
||||
{
|
||||
fprintf(stderr, "Error: dimension mismatch during gemm.\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
int M = A.height(transform_a);
|
||||
int K = A.width(transform_a);
|
||||
int N = B.width(transform_b);
|
||||
|
||||
// Even the host-side implementation utilizes a blocking structure to improve
|
||||
// verification performance
|
||||
int DimBlockM = (M % 16 == 0) ? 16 : 1;
|
||||
int DimBlockN = (N % 16 == 0) ? 16 : 1;
|
||||
|
||||
for (int i = 0; i < M; i += DimBlockM)
|
||||
{
|
||||
for (int j = 0; j < N; j += DimBlockN)
|
||||
{
|
||||
for (int block_y = 0; block_y < DimBlockM; block_y++)
|
||||
{
|
||||
for (int block_x = 0; block_x < DimBlockN; block_x++)
|
||||
{
|
||||
int y = i + block_y;
|
||||
int x = j + block_x;
|
||||
|
||||
host_value_t accum(0);
|
||||
for (int k = 0; k < K; k++)
|
||||
{
|
||||
accum += host_value_t(A.get(k, y, transform_a)) * host_value_t(B.get(x, k, transform_b));
|
||||
}
|
||||
|
||||
_h_data[y + x * M] = (alpha * accum) + (beta * _h_data[y + x * M]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
} // namespace cutlass
|
||||
@@ -0,0 +1,99 @@
|
||||
/******************************************************************************
|
||||
* Copyright (c) 2011-2017, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are not permitted.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
******************************************************************************/
|
||||
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* \file
|
||||
* GPU kernel timer
|
||||
*/
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <cutlass/util/debug.h>
|
||||
|
||||
namespace cutlass {
|
||||
|
||||
|
||||
/******************************************************************************
|
||||
* gpu_timer
|
||||
******************************************************************************/
|
||||
|
||||
/**
|
||||
* GPU event-based timer
|
||||
*/
|
||||
struct gpu_timer
|
||||
{
|
||||
cudaEvent_t _start;
|
||||
cudaEvent_t _stop;
|
||||
|
||||
gpu_timer()
|
||||
{
|
||||
CUDA_PERROR_EXIT(cudaEventCreate(&_start));
|
||||
CUDA_PERROR_EXIT(cudaEventCreate(&_stop));
|
||||
}
|
||||
|
||||
~gpu_timer()
|
||||
{
|
||||
CUDA_PERROR_EXIT(cudaEventDestroy(_start));
|
||||
CUDA_PERROR_EXIT(cudaEventDestroy(_stop));
|
||||
}
|
||||
|
||||
void start()
|
||||
{
|
||||
CUDA_PERROR_EXIT(cudaEventRecord(_start, 0));
|
||||
}
|
||||
|
||||
void stop()
|
||||
{
|
||||
CUDA_PERROR_EXIT(cudaEventRecord(_stop, 0));
|
||||
}
|
||||
|
||||
float elapsed_millis()
|
||||
{
|
||||
float elapsed = 0.0;
|
||||
CUDA_PERROR_EXIT(cudaEventSynchronize(_stop));
|
||||
CUDA_PERROR_EXIT(cudaEventElapsedTime(&elapsed, _start, _stop));
|
||||
return elapsed;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
/******************************************************************************
|
||||
* sleep_millis
|
||||
******************************************************************************/
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
|
||||
void sleep_millis(unsigned milliseconds)
|
||||
{
|
||||
Sleep(milliseconds);
|
||||
}
|
||||
#else
|
||||
#include <unistd.h>
|
||||
|
||||
void sleep_millis(unsigned milliseconds)
|
||||
{
|
||||
usleep(milliseconds * 1000); // takes microseconds
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
} // namespace cutlass
|
||||
@@ -0,0 +1,155 @@
|
||||
/******************************************************************************
|
||||
* Copyright (c) 2011-2017, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are not permitted.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
******************************************************************************/
|
||||
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* \file
|
||||
* \brief Utilities for converting between types and assessing traits
|
||||
*/
|
||||
|
||||
#include "half.h"
|
||||
|
||||
namespace cutlass {
|
||||
|
||||
/******************************************************************************
|
||||
* Float conversion utilities
|
||||
******************************************************************************/
|
||||
|
||||
/// Convert float to value type
|
||||
template <typename value_t>
|
||||
value_t from_float(float val)
|
||||
{
|
||||
return value_t(val);
|
||||
}
|
||||
|
||||
/// Convert float to value type (__half specialization)
|
||||
template <>
|
||||
__half from_float<__half>(float val)
|
||||
{
|
||||
return half_t(val);
|
||||
}
|
||||
|
||||
|
||||
/******************************************************************************
|
||||
* Type conversion utilities
|
||||
******************************************************************************/
|
||||
|
||||
/// Member \p type is defined as the signed integer type having the same size as \p T
|
||||
template <typename T>
|
||||
struct integer_alias;
|
||||
|
||||
template <>
|
||||
struct integer_alias<int8_t> {
|
||||
using type = int8_t;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct integer_alias<half_t> {
|
||||
using type = int16_t;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct integer_alias<__half> {
|
||||
using type = int16_t;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct integer_alias<float> {
|
||||
using type = int32_t;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct integer_alias<int> {
|
||||
using type = int32_t;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct integer_alias<double> {
|
||||
using type = int64_t;
|
||||
};
|
||||
|
||||
|
||||
|
||||
/******************************************************************************
|
||||
* Type-info utilities
|
||||
******************************************************************************/
|
||||
|
||||
/// Returns a string to prefix 'gemm' to construct CUBLAS-like kernel names
|
||||
template <math_operation_class_t math_op, typename value_t, typename accum_t> char const *to_prefix_string();
|
||||
|
||||
template <> char const *to_prefix_string<math_operation_class_t::scalar, half_t, half_t>() {
|
||||
return "H";
|
||||
}
|
||||
|
||||
template <> char const *to_prefix_string<math_operation_class_t::scalar, __half, __half>() {
|
||||
return "H";
|
||||
}
|
||||
|
||||
template <> char const *to_prefix_string<math_operation_class_t::scalar, float, float>() {
|
||||
return "S";
|
||||
}
|
||||
|
||||
template <> char const *to_prefix_string<math_operation_class_t::matrix, __half, __half>() {
|
||||
return "WmmaH";
|
||||
}
|
||||
|
||||
template <> char const *to_prefix_string<math_operation_class_t::matrix, __half, float>() {
|
||||
return "WmmaS";
|
||||
}
|
||||
|
||||
template <> char const *to_prefix_string<math_operation_class_t::scalar, double, double>() {
|
||||
return "D";
|
||||
}
|
||||
|
||||
template <> char const *to_prefix_string<math_operation_class_t::scalar, int8_t, int32_t>() {
|
||||
return "I";
|
||||
}
|
||||
|
||||
|
||||
/******************************************************************************
|
||||
* Maps value_t to the minimum vector size used to load operand
|
||||
******************************************************************************/
|
||||
|
||||
template <typename T>
|
||||
struct operand_load_type;
|
||||
|
||||
template <>
|
||||
struct operand_load_type<int8_t> { using type = int32_t; };
|
||||
|
||||
template <typename T>
|
||||
struct operand_load_type { using type = T; };
|
||||
|
||||
|
||||
/******************************************************************************
|
||||
* Minimum alignment requirement, if any, determined from value_t.
|
||||
******************************************************************************/
|
||||
|
||||
template <typename value_t>
|
||||
struct gemm_alignment_requirement;
|
||||
|
||||
template <>
|
||||
struct gemm_alignment_requirement<uint8_t> { static const int value = 4; };
|
||||
|
||||
template <typename value_t>
|
||||
struct gemm_alignment_requirement { static const int value = 0; };
|
||||
|
||||
|
||||
|
||||
} // namespace cutlass
|
||||
Reference in New Issue
Block a user