CUTLASS v1.0 release
This commit is contained in:
akerr
@@ -0,0 +1,59 @@
|
||||
# Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
# provided that the following conditions are met:
|
||||
# * Redistributions of source code must retain the above copyright notice, this list of
|
||||
# conditions and the following disclaimer.
|
||||
# * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
# conditions and the following disclaimer in the documentation and/or other materials
|
||||
# provided with the distribution.
|
||||
# * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
# to endorse or promote products derived from this software without specific prior written
|
||||
# permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
include_directories(
|
||||
.
|
||||
)
|
||||
|
||||
set(CUTLASS_PERF_TEST_HEADERS
|
||||
testbench_output.h
|
||||
performance_result.h
|
||||
gemm/cublas_dispatch.h
|
||||
gemm/cutlass_dispatch.h
|
||||
gemm/gemm_perf_testbed.h
|
||||
gemm/gemm_profiler.h
|
||||
)
|
||||
|
||||
set(CUTLASS_PERF_TEST_SOURCES
|
||||
cutlass_perf_test.cpp
|
||||
gemm/sgemm.cu
|
||||
gemm/dgemm.cu
|
||||
gemm/hgemm.cu
|
||||
gemm/igemm.cu
|
||||
gemm/wmma_gemm.cu
|
||||
)
|
||||
|
||||
source_group("Source\ Files" FILES ${CUTLASS_PERF_TEST_SOURCES})
|
||||
|
||||
if(NOT CUTLASS_NATIVE_CUDA)
|
||||
# cuda_add_executable does not take interface include directories into account
|
||||
# Let's fetch them and pass them to CUDA.
|
||||
get_target_property(CUTLASS_INCLUDES CUTLASS INTERFACE_INCLUDE_DIRECTORIES)
|
||||
include_directories("${CUTLASS_INCLUDES}")
|
||||
endif()
|
||||
|
||||
cutlass_add_executable(
|
||||
cutlass_perf_test
|
||||
${CUTLASS_PERF_TEST_SOURCES}
|
||||
${CUTLASS_PERF_TEST_HEADERS}
|
||||
)
|
||||
CUDA_ADD_CUBLAS_TO_TARGET(cutlass_perf_test)
|
||||
@@ -0,0 +1,76 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/** \file
|
||||
\brief CUTLASS Performance Tests
|
||||
*/
|
||||
|
||||
#include <tools/test/perf/testbench_options.h>
|
||||
#include <tools/test/perf/testbench_output.h>
|
||||
|
||||
//
|
||||
// Profiling entry points defined in corresponding .cu files
|
||||
//
|
||||
namespace perf {
|
||||
|
||||
int profile_sgemm(TestbenchOutput &output, TestbenchOptions const &options);
|
||||
int profile_dgemm(TestbenchOutput &output, TestbenchOptions const &options);
|
||||
int profile_hgemm(TestbenchOutput &output, TestbenchOptions const &options);
|
||||
int profile_igemm(TestbenchOutput &output, TestbenchOptions const &options);
|
||||
int profile_wmma_gemm(TestbenchOutput &output, TestbenchOptions const &options);
|
||||
|
||||
} // namespace perf
|
||||
|
||||
//
|
||||
// Executes profiling functionality
|
||||
//
|
||||
|
||||
/// Entry point to CUTLASS performance test
|
||||
int main(int argc, const char **argv) {
|
||||
cutlass::CommandLine args(argc, argv);
|
||||
perf::TestbenchOptions options(args);
|
||||
|
||||
if (args.check_cmd_line_flag("help")) {
|
||||
perf::TestbenchOptions::usage(std::cout);
|
||||
return 0;
|
||||
}
|
||||
|
||||
perf::TestbenchOutput output(options);
|
||||
|
||||
int (*profile_gemm[])(perf::TestbenchOutput &, perf::TestbenchOptions const &) = {
|
||||
perf::profile_sgemm,
|
||||
perf::profile_dgemm,
|
||||
perf::profile_hgemm,
|
||||
perf::profile_igemm,
|
||||
perf::profile_wmma_gemm,
|
||||
0};
|
||||
|
||||
int result = 0;
|
||||
for (int i = 0; !result && profile_gemm[i]; ++i) {
|
||||
result = (profile_gemm[i])(output, options);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#include <cutlass/matrix_traits.h>
|
||||
#include <tools/util/type_traits.h>
|
||||
|
||||
namespace perf {
|
||||
|
||||
/// Dispatcher for cuBLAS kernels
|
||||
template <typename AType, typename BType, typename CType, typename Accumulator, typename Scalar>
|
||||
struct CublasGemmDispatch {
|
||||
/// Type used for device-side allocations
|
||||
typedef typename cutlass::TypeTraits<AType>::device_type ADeviceType;
|
||||
typedef typename cutlass::TypeTraits<BType>::device_type BDeviceType;
|
||||
typedef typename cutlass::TypeTraits<CType>::device_type CDeviceType;
|
||||
typedef typename cutlass::TypeTraits<Accumulator>::device_type AccumulatorDeviceType;
|
||||
typedef typename cutlass::TypeTraits<Scalar>::device_type ScalarDeviceType;
|
||||
|
||||
static cublasOperation_t convert(cutlass::MatrixLayout::Kind layout) {
|
||||
switch (layout) {
|
||||
case cutlass::MatrixLayout::kRowMajor:
|
||||
return CUBLAS_OP_T;
|
||||
case cutlass::MatrixLayout::kColumnMajor:
|
||||
return CUBLAS_OP_N;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return CUBLAS_OP_N;
|
||||
}
|
||||
|
||||
/// Launches a cuBLAS GEMM kernel
|
||||
cublasStatus_t operator()(cublasHandle_t handle,
|
||||
cutlass::MatrixLayout::Kind layout_a,
|
||||
cutlass::MatrixLayout::Kind layout_b,
|
||||
int m,
|
||||
int n,
|
||||
int k,
|
||||
Scalar alpha,
|
||||
const ADeviceType *A,
|
||||
int lda,
|
||||
const BDeviceType *B,
|
||||
int ldb,
|
||||
Scalar beta,
|
||||
CDeviceType *C,
|
||||
int ldc,
|
||||
cublasGemmAlgo_t algorithm) {
|
||||
return cublasGemmEx(handle,
|
||||
convert(layout_a),
|
||||
convert(layout_b),
|
||||
m,
|
||||
n,
|
||||
k,
|
||||
reinterpret_cast<ScalarDeviceType const *>(&alpha),
|
||||
A,
|
||||
cutlass::TypeTraits<ADeviceType>::cublas_type,
|
||||
lda,
|
||||
B,
|
||||
cutlass::TypeTraits<BDeviceType>::cublas_type,
|
||||
ldb,
|
||||
reinterpret_cast<ScalarDeviceType const *>(&beta),
|
||||
C,
|
||||
cutlass::TypeTraits<CDeviceType>::cublas_type,
|
||||
ldc,
|
||||
cutlass::TypeTraits<AccumulatorDeviceType>::cublas_type,
|
||||
algorithm);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace perf
|
||||
@@ -0,0 +1,148 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
template <typename Gemm_,
|
||||
typename Index_,
|
||||
typename ScalarA_,
|
||||
typename ScalarB_,
|
||||
typename ScalarC_,
|
||||
typename ScalarD_,
|
||||
typename Compute_,
|
||||
typename ScalarEpilogue_,
|
||||
bool ThreadMultiplyAdd_>
|
||||
struct CutlassDispatch {
|
||||
typedef typename Gemm_::Params Params;
|
||||
typedef Gemm_ Gemm;
|
||||
typedef Index_ Index;
|
||||
typedef ScalarA_ ScalarA;
|
||||
typedef ScalarB_ ScalarB;
|
||||
typedef ScalarC_ ScalarC;
|
||||
typedef ScalarD_ ScalarD;
|
||||
typedef Compute_ Compute;
|
||||
typedef ScalarEpilogue_ ScalarEpilogue;
|
||||
|
||||
static bool const kThreadMultiplyAdd = ThreadMultiplyAdd_;
|
||||
|
||||
static cutlass::MatrixLayout::Kind const kLayoutA = Gemm::Traits::kLayoutA;
|
||||
static cutlass::MatrixLayout::Kind const kLayoutB = Gemm::Traits::kLayoutB;
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// Params argument
|
||||
Params params;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
CutlassDispatch() {}
|
||||
|
||||
/// Initializes params object
|
||||
CutlassDispatch(Index m,
|
||||
Index n,
|
||||
Index k,
|
||||
ScalarEpilogue alpha,
|
||||
ScalarA const* d_a,
|
||||
Index lda,
|
||||
ScalarB const* d_b,
|
||||
Index ldb,
|
||||
ScalarEpilogue beta,
|
||||
ScalarC const* d_c,
|
||||
Index ldc,
|
||||
ScalarD* d_d,
|
||||
Index ldd) {
|
||||
params.initialize(m, n, k, alpha, d_a, lda, d_b, ldb, beta, d_c, ldc, d_d, ldd);
|
||||
}
|
||||
|
||||
/// Initializes params object
|
||||
CutlassDispatch(Params const& _params) : params(_params) {}
|
||||
|
||||
/// Launches kernel
|
||||
cudaError_t operator()() { return Gemm::launch(params); }
|
||||
|
||||
/// Determines if problem is aligned (assuming no padding)
|
||||
static bool is_problem_aligned(
|
||||
int m,
|
||||
int n,
|
||||
int k) {
|
||||
|
||||
bool aligned = true;
|
||||
|
||||
if (kLayoutA == cutlass::MatrixLayout::kColumnMajor) {
|
||||
aligned = aligned && !(m % Gemm::Traits::GemmConfig::kScalarsPerLdgA);
|
||||
}
|
||||
else {
|
||||
aligned = aligned && !(k % Gemm::Traits::GemmConfig::kScalarsPerLdgA);
|
||||
}
|
||||
|
||||
if (kLayoutB == cutlass::MatrixLayout::kColumnMajor) {
|
||||
aligned = aligned && !(k % Gemm::Traits::GemmConfig::kScalarsPerLdgB);
|
||||
}
|
||||
else {
|
||||
aligned = aligned && !(n % Gemm::Traits::GemmConfig::kScalarsPerLdgB);
|
||||
}
|
||||
|
||||
aligned = aligned && !(m % Gemm::Traits::GemmConfig::kScalarsPerLdgC);
|
||||
|
||||
return aligned;
|
||||
}
|
||||
};
|
||||
|
||||
/// Basic dispatcher inferred from GEMM traits
|
||||
template <typename Traits>
|
||||
struct CutlassDispatchBasic {
|
||||
/// Gemm kernel
|
||||
typedef cutlass::gemm::Gemm<Traits> Gemm;
|
||||
|
||||
/// Index type
|
||||
typedef typename Traits::Index Index;
|
||||
|
||||
/// The scalar for A.
|
||||
typedef typename Traits::ScalarA ScalarA;
|
||||
/// The scalar for B.
|
||||
typedef typename Traits::ScalarB ScalarB;
|
||||
/// The scalar for C.
|
||||
typedef typename Traits::ScalarC ScalarC;
|
||||
/// The scalar for D.
|
||||
typedef typename Traits::ScalarD ScalarD;
|
||||
|
||||
// TODO - support alternative accumulator and scalar types
|
||||
typedef ScalarD Compute;
|
||||
typedef Compute ScalarEpilogue;
|
||||
|
||||
typedef CutlassDispatch<Gemm,
|
||||
Index,
|
||||
ScalarA,
|
||||
ScalarB,
|
||||
ScalarC,
|
||||
ScalarD,
|
||||
Compute,
|
||||
ScalarEpilogue,
|
||||
true>
|
||||
Dispatch;
|
||||
};
|
||||
@@ -0,0 +1,97 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/dgemm_traits.h>
|
||||
|
||||
#include <tools/test/perf/gemm/gemm_perf_testbed.h>
|
||||
|
||||
#include <tools/test/perf/gemm/gemm_profiler.h>
|
||||
#include <tools/test/perf/gemm/cutlass_dispatch.h>
|
||||
|
||||
namespace perf {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
int profile_dgemm(TestbenchOutput &output, TestbenchOptions const &options) {
|
||||
|
||||
typedef perf::GemmProfiler<double, double, double, double, double> GemmProfiler;
|
||||
|
||||
int results = 0;
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::DgemmTraits<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor
|
||||
> GemmTraits;
|
||||
|
||||
typedef typename CutlassDispatchBasic<GemmTraits>::Dispatch Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, GemmProfiler>(output, "dgemm_nt", options);
|
||||
}
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::DgemmTraits<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor
|
||||
> GemmTraits;
|
||||
|
||||
typedef typename CutlassDispatchBasic<GemmTraits>::Dispatch Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, GemmProfiler>(output, "dgemm_nn", options);
|
||||
}
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::DgemmTraits<
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor
|
||||
> GemmTraits;
|
||||
|
||||
typedef typename CutlassDispatchBasic<GemmTraits>::Dispatch Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, GemmProfiler>(output, "dgemm_tn", options);
|
||||
}
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::DgemmTraits<
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor
|
||||
> GemmTraits;
|
||||
|
||||
typedef typename CutlassDispatchBasic<GemmTraits>::Dispatch Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, GemmProfiler>(output, "dgemm_tt", options);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace perf
|
||||
@@ -0,0 +1,624 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
// Standard Library includes
|
||||
#include <fstream>
|
||||
#include <ostream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
// CUDA includes
|
||||
#include <cublas_v2.h>
|
||||
#include <curand_kernel.h>
|
||||
|
||||
// Cutlass includes
|
||||
#include <tools/test/perf/gemm/cublas_dispatch.h>
|
||||
#include <tools/test/perf/performance_result.h>
|
||||
#include <tools/test/perf/testbench_options.h>
|
||||
#include <tools/util/device_memory.h>
|
||||
#include <tools/util/type_traits.h>
|
||||
#include <tools/util/host_tensor.h>
|
||||
#include <tools/util/tensor_view_io.h>
|
||||
|
||||
namespace perf {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Kernel to determine if two tensors are equal
|
||||
template <typename Type>
|
||||
__global__ void tensor_equals(int *result,
|
||||
int dim_contiguous,
|
||||
int dim_strided,
|
||||
Type const *experimental,
|
||||
int lde,
|
||||
Type const *reference,
|
||||
int ldr) {
|
||||
typedef typename cutlass::TypeTraits<Type>::unsigned_type UnsignedType;
|
||||
|
||||
int c_idx = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int s_idx = blockIdx.y * blockDim.x;
|
||||
|
||||
experimental += s_idx * lde + c_idx;
|
||||
reference += s_idx * ldr + c_idx;
|
||||
|
||||
for (int s_offset = 0; s_offset < blockDim.x; ++s_offset, ++s_idx) {
|
||||
if (s_idx < dim_strided && c_idx < dim_contiguous) {
|
||||
UnsignedType exp = *reinterpret_cast<UnsignedType const *>(experimental);
|
||||
UnsignedType ref = *reinterpret_cast<UnsignedType const *>(reference);
|
||||
|
||||
if (exp != ref) {
|
||||
*result = -1;
|
||||
return;
|
||||
}
|
||||
|
||||
experimental += lde;
|
||||
reference += ldr;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Kernel to initialize tensor to uniform distribution
|
||||
template <typename T>
|
||||
__global__ void initialize_uniform(
|
||||
Distribution dist, int64_t seed, int dim_contiguous, int dim_strided, T *tensor, int ldm) {
|
||||
__shared__ curandState_t rng_state[1024];
|
||||
|
||||
uint64_t gtid = threadIdx.x + blockIdx.x * blockDim.x + blockIdx.y * gridDim.x * blockDim.x;
|
||||
|
||||
curand_init(seed, gtid, 0, &rng_state[threadIdx.x]);
|
||||
|
||||
int c_idx = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int s_idx = blockIdx.y * blockDim.x;
|
||||
|
||||
tensor += s_idx * ldm + c_idx;
|
||||
|
||||
for (int s_offset = 0; s_offset < blockDim.x; ++s_offset, ++s_idx) {
|
||||
if (s_idx < dim_strided && c_idx < dim_contiguous) {
|
||||
double range = dist.uniform.max - dist.uniform.min;
|
||||
|
||||
double rnd = curand_uniform(&rng_state[threadIdx.x]);
|
||||
|
||||
rnd = dist.uniform.min + range * rnd;
|
||||
|
||||
// Random values are cast to integer after scaling by a power of two to facilitate error
|
||||
// testing
|
||||
if (dist.int_scale >= 0) {
|
||||
rnd = double(int(rnd * double(1 << dist.int_scale)));
|
||||
*tensor = T(rnd / double(1 << dist.int_scale));
|
||||
} else {
|
||||
*tensor = T(rnd);
|
||||
}
|
||||
|
||||
tensor += ldm;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Kernel to initialize tensor to uniform distribution
|
||||
template <typename T>
|
||||
__global__ void initialize_gaussian(
|
||||
Distribution dist, int64_t seed, int dim_contiguous, int dim_strided, T *tensor, int ldm) {
|
||||
__shared__ curandState_t rng_state[1024];
|
||||
|
||||
uint64_t gtid = threadIdx.x + blockIdx.x * blockDim.x + blockIdx.y * gridDim.x * blockDim.x;
|
||||
|
||||
curand_init(seed, gtid, 0, &rng_state[threadIdx.x]);
|
||||
|
||||
int c_idx = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int s_idx = blockIdx.y * blockDim.x;
|
||||
|
||||
tensor += s_idx * ldm + c_idx;
|
||||
|
||||
for (int s_offset = 0; s_offset < blockDim.x; ++s_offset, ++s_idx) {
|
||||
if (s_idx < dim_strided && c_idx < dim_contiguous) {
|
||||
// Random values are cast to integer after scaling by a power of two to facilitate error
|
||||
// testing
|
||||
|
||||
double rnd = curand_normal(&rng_state[threadIdx.x]);
|
||||
|
||||
rnd = dist.gaussian.mean + dist.gaussian.stddev * rnd;
|
||||
|
||||
if (dist.int_scale >= 0) {
|
||||
rnd = double(int(rnd * double(1 << dist.int_scale)));
|
||||
*tensor = T(rnd / double(1 << dist.int_scale));
|
||||
} else {
|
||||
*tensor = T(rnd);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Kernel to initialize tensor to an identity matrix
|
||||
template <typename T>
|
||||
__global__ void initialize_linear(
|
||||
Distribution dist, int64_t seed, int dim_contiguous, int dim_strided, T *tensor, int ldm) {
|
||||
__shared__ curandState_t rng_state[1024];
|
||||
|
||||
uint64_t gtid = threadIdx.x + blockIdx.x * blockDim.x + blockIdx.y * gridDim.x * blockDim.x;
|
||||
|
||||
curand_init(seed, gtid, 0, &rng_state[threadIdx.x]);
|
||||
|
||||
int c_idx = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int s_idx = blockIdx.y * blockDim.x;
|
||||
|
||||
tensor += s_idx * ldm + c_idx;
|
||||
|
||||
for (int s_offset = 0; s_offset < blockDim.x; ++s_offset, ++s_idx) {
|
||||
if (s_idx < dim_strided && c_idx < dim_contiguous) {
|
||||
*tensor =
|
||||
dist.linear.offset + dist.linear.delta_row * c_idx + dist.linear.delta_column * s_idx;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Kernel to initialize tensor to an identity matrix
|
||||
template <typename T>
|
||||
__global__ void initialize_identity(
|
||||
Distribution dist, int64_t seed, int dim_contiguous, int dim_strided, T *tensor, int ldm) {
|
||||
__shared__ curandState_t rng_state[1024];
|
||||
|
||||
uint64_t gtid = threadIdx.x + blockIdx.x * blockDim.x + blockIdx.y * gridDim.x * blockDim.x;
|
||||
|
||||
curand_init(seed, gtid, 0, &rng_state[threadIdx.x]);
|
||||
|
||||
int c_idx = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int s_idx = blockIdx.y * blockDim.x;
|
||||
|
||||
tensor += s_idx * ldm + c_idx;
|
||||
|
||||
for (int s_offset = 0; s_offset < blockDim.x; ++s_offset, ++s_idx) {
|
||||
if (s_idx < dim_strided && c_idx < dim_contiguous) {
|
||||
*tensor = (c_idx == s_idx ? T(1) : T(0));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Dispatcher to appropriate initialization kernel
|
||||
template <typename T>
|
||||
inline void initialize(Distribution const &dist,
|
||||
int64_t seed,
|
||||
int dim_contiguous,
|
||||
int dim_strided,
|
||||
T *tensor,
|
||||
int ldm) {
|
||||
dim3 block(256, 1, 1);
|
||||
dim3 grid((dim_contiguous + block.x - 1) / block.x, (dim_strided + block.x - 1) / block.x);
|
||||
|
||||
switch (dist.kind) {
|
||||
case Distribution::Uniform:
|
||||
initialize_uniform<<<grid, block>>>(dist, seed, dim_contiguous, dim_strided, tensor, ldm);
|
||||
break;
|
||||
case Distribution::Gaussian:
|
||||
initialize_gaussian<<<grid, block>>>(dist, seed, dim_contiguous, dim_strided, tensor, ldm);
|
||||
break;
|
||||
case Distribution::Linear:
|
||||
initialize_linear<<<grid, block>>>(dist, seed, dim_contiguous, dim_strided, tensor, ldm);
|
||||
break;
|
||||
case Distribution::Identity:
|
||||
initialize_identity<<<grid, block>>>(dist, seed, dim_contiguous, dim_strided, tensor, ldm);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Host-side implementation of performance testbed
|
||||
template <typename AType, typename BType, typename CType, typename Accumulator, typename Scalar>
|
||||
class GemmTestbed {
|
||||
public:
|
||||
/// Type used for device-side allocations
|
||||
typedef typename cutlass::TypeTraits<AType>::device_type ADeviceType;
|
||||
typedef typename cutlass::TypeTraits<BType>::device_type BDeviceType;
|
||||
typedef typename cutlass::TypeTraits<CType>::device_type CDeviceType;
|
||||
typedef typename cutlass::TypeTraits<Accumulator>::device_type AccumulatorDeviceType;
|
||||
typedef typename cutlass::TypeTraits<Scalar>::device_type ScalarDeviceType;
|
||||
|
||||
/// Dispatch object to cuBLAS GEMM
|
||||
typedef CublasGemmDispatch<AType, BType, CType, Accumulator, Scalar> CublasDispatch;
|
||||
|
||||
//
|
||||
// Type definitions
|
||||
//
|
||||
|
||||
/// Host tensor for operand A
|
||||
typedef cutlass::device_memory::allocation<ADeviceType> TensorA;
|
||||
|
||||
/// Host tensor for operand B
|
||||
typedef cutlass::device_memory::allocation<BDeviceType> TensorB;
|
||||
|
||||
/// Host tensor for operand C
|
||||
typedef cutlass::device_memory::allocation<CDeviceType> TensorC;
|
||||
|
||||
private:
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
InitialDistribution initial_distribution;
|
||||
|
||||
/// Status
|
||||
cublasStatus_t status;
|
||||
|
||||
/// cuBLAS handle
|
||||
cublasHandle_t handle;
|
||||
|
||||
/// GEMM problem
|
||||
GemmProblem problem;
|
||||
|
||||
/// A matrix operand
|
||||
TensorA A;
|
||||
|
||||
/// B matrix operand
|
||||
TensorB B;
|
||||
|
||||
/// C matrix operand
|
||||
TensorC C_initial;
|
||||
|
||||
/// Reference result
|
||||
TensorC reference;
|
||||
|
||||
/// Experimental result
|
||||
TensorC experimental;
|
||||
|
||||
private:
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Helper to resize a matrix with a given size and layout if needed
|
||||
template <typename T>
|
||||
static void resize_device_allocation(
|
||||
cutlass::device_memory::allocation<T> &tensor,
|
||||
Distribution const &dist,
|
||||
int64_t seed,
|
||||
int rows,
|
||||
int columns,
|
||||
cutlass::MatrixLayout::Kind layout,
|
||||
int ldm = 0) {
|
||||
if (!ldm) {
|
||||
ldm = (layout == cutlass::MatrixLayout::kColumnMajor ? rows : columns);
|
||||
}
|
||||
|
||||
size_t capacity = ldm * (layout == cutlass::MatrixLayout::kColumnMajor ? columns : rows);
|
||||
|
||||
if (capacity > tensor.capacity) {
|
||||
tensor.reset(cutlass::device_memory::allocate<T>(capacity), capacity);
|
||||
|
||||
int c_dim = (layout == cutlass::MatrixLayout::kColumnMajor ? rows : columns);
|
||||
int s_dim = (layout == cutlass::MatrixLayout::kColumnMajor ? columns : rows);
|
||||
|
||||
initialize(dist, seed, c_dim, s_dim, tensor.get(), ldm);
|
||||
}
|
||||
}
|
||||
|
||||
/// Resizes each tensor
|
||||
void resize_helper(GemmProblem const &problem) {
|
||||
resize_device_allocation(
|
||||
A,
|
||||
initial_distribution.dist_A,
|
||||
initial_distribution.seed,
|
||||
problem.m,
|
||||
problem.k,
|
||||
problem.layout_A);
|
||||
|
||||
resize_device_allocation(
|
||||
B,
|
||||
initial_distribution.dist_B,
|
||||
initial_distribution.seed + 17, // compute distinct value from initial seed
|
||||
problem.k,
|
||||
problem.n,
|
||||
problem.layout_B);
|
||||
|
||||
resize_device_allocation(
|
||||
C_initial,
|
||||
initial_distribution.dist_C,
|
||||
initial_distribution.seed + 101, // compute distinct value from initial seed
|
||||
problem.m,
|
||||
problem.n,
|
||||
cutlass::MatrixLayout::kColumnMajor);
|
||||
|
||||
resize_device_allocation(
|
||||
reference, Distribution(), 0, problem.m, problem.n, cutlass::MatrixLayout::kColumnMajor);
|
||||
|
||||
resize_device_allocation(
|
||||
experimental, Distribution(), 0, problem.m, problem.n, cutlass::MatrixLayout::kColumnMajor);
|
||||
}
|
||||
|
||||
/// Functor to print errors
|
||||
struct PrintErrors {
|
||||
|
||||
/// Equivalently sized integer type
|
||||
typedef typename cutlass::TypeTraits<CType>::integer_type integer_t;
|
||||
|
||||
/// Output stream to write to
|
||||
std::ostream& out;
|
||||
|
||||
/// Reference tensor view
|
||||
cutlass::HostTensorView<CType> const& reference;
|
||||
|
||||
/// Computed tensor view
|
||||
cutlass::HostTensorView<CType> const& experimental;
|
||||
|
||||
/// Errors greater than or this amount result in printing
|
||||
integer_t ulps_threshold;
|
||||
|
||||
///
|
||||
PrintErrors(std::ostream& _out,
|
||||
cutlass::HostTensorView<CType> const& _reference,
|
||||
cutlass::HostTensorView<CType> const& _experimental,
|
||||
integer_t _ulps_threshold = 1)
|
||||
: out(_out),
|
||||
reference(_reference),
|
||||
experimental(_experimental),
|
||||
ulps_threshold(_ulps_threshold) {}
|
||||
|
||||
/// Compares one element
|
||||
void operator()(
|
||||
CType const& element,
|
||||
typename cutlass::HostTensorView<CType>::Coord_t coord) {
|
||||
|
||||
CType exp = experimental.at(coord);
|
||||
CType ref = reference.at(coord);
|
||||
|
||||
int64_t int_exp = 0;
|
||||
int64_t int_ref = 0;
|
||||
|
||||
*reinterpret_cast<CType*>(&int_exp) = exp;
|
||||
*reinterpret_cast<CType*>(&int_ref) = ref;
|
||||
|
||||
integer_t ulps = integer_t(int_exp - int_ref);
|
||||
|
||||
if (std::abs(ulps) >= ulps_threshold) {
|
||||
// width in hexadecimal digits of value
|
||||
int const width = sizeof(integer_t) * 2;
|
||||
|
||||
double relative = double(exp) - double(ref);
|
||||
if (ref != CType(0)) {
|
||||
relative /= double(ref);
|
||||
}
|
||||
|
||||
out << "[" << coord << "] expected: " << ref << " (0x"
|
||||
<< std::hex << std::setw(width) << std::setfill('0') << integer_t(int_ref) << std::dec
|
||||
<< ")"
|
||||
<< ", got: " << exp << " (0x" << std::hex
|
||||
<< std::setw(width) << std::setfill('0') << integer_t(int_exp) << std::dec << ")"
|
||||
<< " relative error: " << relative << ", ulps: " << ulps << "\n";
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
public:
|
||||
/// Resizes tensors to accommodate the given problem
|
||||
void resize(GemmProblem const &_problem) {
|
||||
problem = _problem;
|
||||
|
||||
try {
|
||||
resize_helper(problem);
|
||||
} catch (...) {
|
||||
// If out of memory, clear each allocation then allocate again
|
||||
A.reset();
|
||||
B.reset();
|
||||
C_initial.reset();
|
||||
reference.reset();
|
||||
experimental.reset();
|
||||
|
||||
resize_helper(problem);
|
||||
}
|
||||
}
|
||||
|
||||
/// Constructs a basic workspace
|
||||
GemmTestbed(InitialDistribution const &_dist = InitialDistribution())
|
||||
: initial_distribution(_dist) {
|
||||
status = cublasCreate(&handle);
|
||||
if (status != CUBLAS_STATUS_SUCCESS) {
|
||||
throw cutlass::cuda_exception("Failed to create CUBLAS handle");
|
||||
}
|
||||
}
|
||||
|
||||
/// Constructs a workspace for verifying GEMM, assumes
|
||||
/// dense packing.
|
||||
GemmTestbed(GemmProblem const &_problem,
|
||||
cublasGemmAlgo_t algorithm_ = CUBLAS_GEMM_DEFAULT,
|
||||
InitialDistribution const &_dist = InitialDistribution())
|
||||
: problem(_problem), initial_distribution(_dist) {
|
||||
status = cublasCreate(&handle);
|
||||
if (status != CUBLAS_STATUS_SUCCESS) {
|
||||
throw cutlass::cuda_exception("Failed to create CUBLAS handle");
|
||||
}
|
||||
|
||||
resize(problem);
|
||||
}
|
||||
|
||||
~GemmTestbed() { status = cublasDestroy(handle); }
|
||||
|
||||
/// Returns true if the last CUBLAS call returned successfully
|
||||
bool good() const { return status == CUBLAS_STATUS_SUCCESS; }
|
||||
|
||||
/// Rows of GEMM problem
|
||||
int M() const { return problem.m; }
|
||||
|
||||
/// Columns of GEMM problem
|
||||
int N() const { return problem.n; }
|
||||
|
||||
/// Inner dimension of GEMM problem
|
||||
int K() const { return problem.k; }
|
||||
|
||||
/// Returns a pointer to the A operand
|
||||
ADeviceType *ptr_A() const { return A.get(); }
|
||||
|
||||
/// Leading dimension of A
|
||||
int lda() const { return problem.lda(); }
|
||||
|
||||
/// Returns a pointer to the B operand
|
||||
BDeviceType *ptr_B() const { return B.get(); }
|
||||
|
||||
/// Leading dimension of B
|
||||
int ldb() const { return problem.ldb(); }
|
||||
|
||||
/// Returns a pointer to the initial state of the result tensor in device memory
|
||||
CDeviceType *ptr_C_initial() const { return C_initial.get(); }
|
||||
|
||||
/// Leading dimension of C
|
||||
int ldc() const { return problem.ldc(); }
|
||||
|
||||
/// Returns a pointer to the result tensor in device memory
|
||||
CDeviceType *ptr_experimental() const { return experimental.get(); }
|
||||
|
||||
/// Returns a pointer to the result tensor in device memory
|
||||
CDeviceType *ptr_reference() const { return reference.get(); }
|
||||
|
||||
/// Returns the number of flops implied by the computation (1 multiply-accumulate = 2 flops)
|
||||
uint64_t flops() const {
|
||||
return uint64_t(problem.m) * uint64_t(problem.n) * uint64_t(problem.k) * 2ULL;
|
||||
}
|
||||
|
||||
/// Computes the speed of the computation in GFLOPs/s
|
||||
double GFLOPs_per_sec(double runtime_ms) const { return double(flops()) / runtime_ms / 1.0e6; }
|
||||
|
||||
/// Matrix layout of A
|
||||
cutlass::MatrixLayout::Kind layout_a() const { return problem.layout_A; }
|
||||
|
||||
/// Matrix layout of B
|
||||
cutlass::MatrixLayout::Kind layout_b() const { return problem.layout_B; }
|
||||
|
||||
/// Returns alpha scalar
|
||||
Scalar alpha() const { return Scalar(problem.alpha); }
|
||||
|
||||
/// Returns alpha scalar
|
||||
Scalar beta() const { return Scalar(problem.beta); }
|
||||
|
||||
/// Initializes C matrix by copying from C_initial
|
||||
void prepare_gemm(CDeviceType *target) {
|
||||
size_t count = ldc() * problem.n;
|
||||
cutlass::device_memory::copy_device_to_device(target, ptr_C_initial(), count);
|
||||
}
|
||||
|
||||
/// Initializes output matrix of cublas
|
||||
void prepare_cublas() { prepare_gemm(ptr_reference()); }
|
||||
|
||||
/// Initializes output matrix of cublas
|
||||
void prepare_experimental() { prepare_gemm(ptr_experimental()); }
|
||||
|
||||
/// Launches the cuBLAS GEMM - does not initialize output matrix
|
||||
cublasStatus_t launch_cublas(cublasGemmAlgo_t algo) {
|
||||
CublasDispatch dispatch;
|
||||
|
||||
Scalar alpha(Scalar(problem.alpha));
|
||||
Scalar beta(Scalar(problem.beta));
|
||||
|
||||
status = dispatch(handle,
|
||||
problem.layout_A,
|
||||
problem.layout_B,
|
||||
problem.m,
|
||||
problem.n,
|
||||
problem.k,
|
||||
alpha,
|
||||
ptr_A(),
|
||||
lda(),
|
||||
ptr_B(),
|
||||
ldb(),
|
||||
beta,
|
||||
ptr_reference(),
|
||||
ldc(),
|
||||
algo);
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
/// Verifies the 'test' tensor with 'ref'
|
||||
bool verify(TensorC const &test, TensorC const &ref) {
|
||||
cutlass::device_memory::allocation<int> flag_device(1);
|
||||
|
||||
int flag = 0;
|
||||
cutlass::device_memory::copy_to_device(flag_device.get(), &flag, 1);
|
||||
|
||||
dim3 block(256, 1, 1);
|
||||
dim3 grid((problem.m + block.x - 1) / block.x, (problem.n + block.x - 1) / block.x);
|
||||
|
||||
tensor_equals<CDeviceType><<<grid, block>>>(flag_device.get(),
|
||||
problem.m,
|
||||
problem.n,
|
||||
experimental.get(),
|
||||
problem.m,
|
||||
reference.get(),
|
||||
problem.m);
|
||||
|
||||
cutlass::device_memory::copy_to_host(&flag, flag_device.get(), 1);
|
||||
|
||||
return flag == 0;
|
||||
}
|
||||
|
||||
/// Computes the reference output
|
||||
void compute_reference(cublasGemmAlgo_t algorithm) {
|
||||
prepare_cublas();
|
||||
launch_cublas(algorithm);
|
||||
}
|
||||
|
||||
/// Helper to verify with reference
|
||||
bool verify_with_reference() { return verify(experimental, reference); }
|
||||
|
||||
/// Writes the problem to an ostream in human-readable form
|
||||
void write_problem(std::ostream &results_output, std::ostream &errors_output) {
|
||||
|
||||
cutlass::HostTensor<AType, false> host_A;
|
||||
cutlass::HostTensor<BType, false> host_B;
|
||||
cutlass::HostTensor<CType, false> host_C;
|
||||
cutlass::HostTensor<CType, false> host_D;
|
||||
cutlass::HostTensor<CType, false> host_Ref;
|
||||
|
||||
host_A.resize_matrix(M(), K(), layout_a());
|
||||
host_B.resize_matrix(K(), N(), layout_b());
|
||||
host_C.resize_matrix(M(), N(), cutlass::MatrixLayout::kColumnMajor);
|
||||
host_D.resize_matrix(M(), N(), cutlass::MatrixLayout::kColumnMajor);
|
||||
host_Ref.resize_matrix(M(), N(), cutlass::MatrixLayout::kColumnMajor);
|
||||
|
||||
// copy from device allocations
|
||||
host_A.copy_to_host(ptr_A());
|
||||
host_B.copy_to_host(ptr_B());
|
||||
host_C.copy_to_host(ptr_C_initial());
|
||||
host_D.copy_to_host(ptr_experimental());
|
||||
host_Ref.copy_to_host(ptr_reference());
|
||||
|
||||
// write out human readable
|
||||
results_output << "A =\n" << host_A << "\n"
|
||||
<< "B =\n" << host_B << "\n"
|
||||
<< "C = \n" << host_C << "\n"
|
||||
<< "Ref =\n" << host_Ref << "\n"
|
||||
<< "Experimental =\n" << host_D << "\n";
|
||||
|
||||
// write out list of errors
|
||||
PrintErrors printer(errors_output, host_Ref, host_D);
|
||||
|
||||
host_D.visit(printer);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace perf
|
||||
@@ -0,0 +1,343 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <stdexcept>
|
||||
#include <utility>
|
||||
|
||||
#if defined(WIN32)
|
||||
#include <Windows.h>
|
||||
#else
|
||||
// needed for sleep
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#include <tools/test/perf/gemm/gemm_perf_testbed.h>
|
||||
#include <tools/test/perf/testbench_options.h>
|
||||
#include <tools/test/perf/testbench_output.h>
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace perf {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Performance measuring testbed
|
||||
template <typename AType,
|
||||
typename BType,
|
||||
typename CType,
|
||||
typename AccumulatorType,
|
||||
typename ScalarType>
|
||||
class GemmProfiler {
|
||||
public:
|
||||
/// Test environment
|
||||
typedef GemmTestbed<AType, BType, CType, AccumulatorType, ScalarType> PerfTestbed;
|
||||
|
||||
private:
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// Reference to TestbenchOutput instance
|
||||
TestbenchOutput &output;
|
||||
|
||||
/// Reference to options object
|
||||
TestbenchOptions const &options;
|
||||
|
||||
/// Performance test environment
|
||||
PerfTestbed testbed;
|
||||
|
||||
/// Kernel name
|
||||
std::string kernel_name;
|
||||
|
||||
/// Timing events
|
||||
cudaEvent_t events[2];
|
||||
|
||||
public:
|
||||
/// Delays
|
||||
static void pause(int seconds) {
|
||||
#if defined(WIN32)
|
||||
Sleep(1000 * seconds);
|
||||
#else
|
||||
sleep(seconds);
|
||||
#endif
|
||||
}
|
||||
|
||||
public:
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Constructs performance testebed
|
||||
GemmProfiler(TestbenchOutput &_output,
|
||||
std::string const &_kernel_name,
|
||||
TestbenchOptions const &_options)
|
||||
: output(_output),
|
||||
options(_options),
|
||||
kernel_name(_kernel_name),
|
||||
testbed(_options.initial_distribution) {
|
||||
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
cudaError_t result = cudaEventCreate(&events[i]);
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("GemmPerfTestbed() failed to create CUDA events");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
~GemmProfiler() {}
|
||||
|
||||
/// Writes the workspace to text files
|
||||
void write_problem(std::string const &kernel_name) {
|
||||
|
||||
std::stringstream base_filename;
|
||||
|
||||
base_filename
|
||||
<< kernel_name << "_"
|
||||
<< testbed.M() << "x" << testbed.N() << "x" << testbed.K();
|
||||
|
||||
std::string results_name = base_filename.str() + "_results.txt";
|
||||
std::string errors_name = base_filename.str() + "_errors.txt";
|
||||
|
||||
std::ofstream results(results_name.c_str());
|
||||
std::ofstream errors(errors_name.c_str());
|
||||
testbed.write_problem(results, errors);
|
||||
}
|
||||
|
||||
/// Profiles Cutlass
|
||||
template <typename CutlassDispatch>
|
||||
PerformanceResult execute_cutlass(GemmProblem const &problem, cublasGemmAlgo_t algorithm) {
|
||||
PerformanceResult result(kernel_name, problem);
|
||||
|
||||
testbed.compute_reference(algorithm);
|
||||
|
||||
if (cudaDeviceSynchronize() != cudaSuccess) {
|
||||
result.disposition = Disposition::NotVerified;
|
||||
return result;
|
||||
}
|
||||
|
||||
CutlassDispatch dispatch(testbed.M(),
|
||||
testbed.N(),
|
||||
testbed.K(),
|
||||
testbed.alpha(),
|
||||
testbed.ptr_A(),
|
||||
testbed.lda(),
|
||||
testbed.ptr_B(),
|
||||
testbed.ldb(),
|
||||
testbed.beta(),
|
||||
testbed.ptr_C_initial(),
|
||||
testbed.ldc(),
|
||||
testbed.ptr_experimental(),
|
||||
testbed.ldc());
|
||||
|
||||
dispatch();
|
||||
|
||||
if (cudaDeviceSynchronize() != cudaSuccess) {
|
||||
result.disposition = Disposition::Failed;
|
||||
return result;
|
||||
}
|
||||
|
||||
if (testbed.verify_with_reference()) {
|
||||
result.disposition = Disposition::Passed;
|
||||
} else {
|
||||
result.disposition = Disposition::Incorrect;
|
||||
}
|
||||
|
||||
if (options.save_workspace(result.disposition == Disposition::Passed)) {
|
||||
write_problem(kernel_name);
|
||||
}
|
||||
|
||||
if (cudaDeviceSynchronize() != cudaSuccess) {
|
||||
result.disposition = Disposition::Failed;
|
||||
}
|
||||
|
||||
// warmup launch
|
||||
dispatch();
|
||||
|
||||
if (cudaDeviceSynchronize() != cudaSuccess) {
|
||||
result.disposition = Disposition::Failed;
|
||||
return result;
|
||||
}
|
||||
|
||||
if (cudaEventRecord(events[0]) != cudaSuccess) {
|
||||
result.disposition = Disposition::Failed;
|
||||
return result;
|
||||
}
|
||||
|
||||
for (int iter = 0; iter < options.iterations; ++iter) {
|
||||
dispatch();
|
||||
}
|
||||
|
||||
if (cudaEventRecord(events[1]) != cudaSuccess) {
|
||||
result.disposition = Disposition::Failed;
|
||||
return result;
|
||||
}
|
||||
|
||||
if (cudaEventSynchronize(events[1]) != cudaSuccess) {
|
||||
result.disposition = Disposition::Failed;
|
||||
return result;
|
||||
}
|
||||
|
||||
float average_ms = 0;
|
||||
if (cudaEventElapsedTime(&average_ms, events[0], events[1]) != cudaSuccess) {
|
||||
result.disposition = Disposition::Failed;
|
||||
return result;
|
||||
}
|
||||
|
||||
result.runtime = double(average_ms) / double(options.iterations);
|
||||
result.gflops = testbed.GFLOPs_per_sec(result.runtime);
|
||||
|
||||
if (result.disposition != Disposition::Passed) {
|
||||
std::cout << kernel_name << " failed with disposition: " << result.disposition;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// Executes all kernels for this problem size
|
||||
template <typename CutlassDispatch>
|
||||
std::vector<PerformanceResult> execute(GemmProblem const &problem) {
|
||||
|
||||
// New problem size
|
||||
output.begin_problem();
|
||||
|
||||
cublasGemmAlgo_t algorithm =
|
||||
(CutlassDispatch::kThreadMultiplyAdd ? CUBLAS_GEMM_DEFAULT : CUBLAS_GEMM_DEFAULT_TENSOR_OP);
|
||||
|
||||
testbed.resize(problem);
|
||||
|
||||
std::vector<PerformanceResult> results;
|
||||
|
||||
results.push_back(execute_cutlass<CutlassDispatch>(problem, algorithm));
|
||||
|
||||
// cool-down period
|
||||
pause(2);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/// Runs the test and collects performance for all results
|
||||
template <typename CutlassDispatch>
|
||||
void schmoo(Range const &M, Range const &N, Range const &K) {
|
||||
for (int m = M.start; m <= M.end; m += M.increment) {
|
||||
for (int n = N.start; n <= N.end; n += N.increment) {
|
||||
for (int k = K.start; k <= K.end; k += K.increment) {
|
||||
|
||||
// Avoid evaluating problem if problem size does not satisfy alignment
|
||||
if (!CutlassDispatch::is_problem_aligned(m, n, k)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::vector<PerformanceResult> results =
|
||||
execute<CutlassDispatch>(GemmProblem(m,
|
||||
n,
|
||||
k,
|
||||
CutlassDispatch::kLayoutA,
|
||||
CutlassDispatch::kLayoutB,
|
||||
options.alpha,
|
||||
options.beta));
|
||||
|
||||
for (std::vector<PerformanceResult>::const_iterator it = results.begin();
|
||||
it != results.end();
|
||||
++it) {
|
||||
output.append(*it);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs the test over the problem space and reports only the best performance
|
||||
template <typename CutlassDispatch>
|
||||
void peak(Range const &M, Range const &N, Range const &K) {
|
||||
|
||||
PerformanceResult max_perf;
|
||||
bool first_result = true;
|
||||
|
||||
for (int m = M.start; m <= M.end; m += M.increment) {
|
||||
for (int n = N.start; n <= N.end; n += N.increment) {
|
||||
for (int k = K.start; k <= K.end; k += K.increment) {
|
||||
|
||||
// Avoid evaluating problem if problem size does not satisfy alignment
|
||||
if (!CutlassDispatch::is_problem_aligned(m, n, k)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::vector<PerformanceResult> results =
|
||||
execute<CutlassDispatch>(GemmProblem(m,
|
||||
n,
|
||||
k,
|
||||
CutlassDispatch::kLayoutA,
|
||||
CutlassDispatch::kLayoutB,
|
||||
options.alpha,
|
||||
options.beta));
|
||||
|
||||
for (std::vector<PerformanceResult>::const_iterator it = results.begin();
|
||||
it != results.end();
|
||||
++it) {
|
||||
|
||||
/// Writes the output without appending it
|
||||
output.pretty_print(*it);
|
||||
|
||||
/// Updates maximum performing kernel
|
||||
if (first_result || max_perf.gflops > it->gflops) {
|
||||
max_perf = *it;
|
||||
}
|
||||
first_result = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
output.append(max_perf);
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Dispatches to GEMM performance profiler
|
||||
template <typename Dispatch, typename GemmProfiler>
|
||||
int profile_gemm(TestbenchOutput &output,
|
||||
std::string const &kernel,
|
||||
TestbenchOptions const &options) {
|
||||
if (options.kernel_enabled(kernel)) {
|
||||
GemmProfiler perf(output, kernel, options);
|
||||
if (options.peak_performance) {
|
||||
perf.template peak<Dispatch>(
|
||||
options.problem_range.M, options.problem_range.N, options.problem_range.K);
|
||||
} else {
|
||||
perf.template schmoo<Dispatch>(
|
||||
options.problem_range.M, options.problem_range.N, options.problem_range.K);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace perf
|
||||
@@ -0,0 +1,113 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/hgemm_traits.h>
|
||||
|
||||
#include <tools/test/perf/gemm/gemm_perf_testbed.h>
|
||||
|
||||
#include <tools/test/perf/gemm/gemm_profiler.h>
|
||||
#include <tools/test/perf/gemm/cutlass_dispatch.h>
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
namespace perf {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
int profile_hgemm(TestbenchOutput &output, TestbenchOptions const &options) {
|
||||
|
||||
typedef perf::GemmProfiler<
|
||||
cutlass::half_t,
|
||||
cutlass::half_t,
|
||||
cutlass::half_t,
|
||||
cutlass::half_t,
|
||||
cutlass::half_t> GemmProfiler;
|
||||
|
||||
int results = 0;
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::HgemmTraits<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<8, 128, 128>
|
||||
>
|
||||
GemmTraits;
|
||||
|
||||
typedef typename CutlassDispatchBasic<GemmTraits>::Dispatch Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, GemmProfiler>(output, "hgemm_nt", options);
|
||||
}
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::HgemmTraits<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<8, 128, 128>
|
||||
>
|
||||
GemmTraits;
|
||||
|
||||
typedef typename CutlassDispatchBasic<GemmTraits>::Dispatch Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, GemmProfiler>(output, "hgemm_nn", options);
|
||||
}
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::HgemmTraits<
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<8, 128, 128>
|
||||
>
|
||||
GemmTraits;
|
||||
|
||||
typedef typename CutlassDispatchBasic<GemmTraits>::Dispatch Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, GemmProfiler>(output, "hgemm_tn", options);
|
||||
}
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::HgemmTraits<
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<8, 128, 128>
|
||||
>
|
||||
GemmTraits;
|
||||
|
||||
typedef typename CutlassDispatchBasic<GemmTraits>::Dispatch Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, GemmProfiler>(output, "hgemm_tt", options);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace perf
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/igemm_traits.h>
|
||||
#include <tools/test/perf/gemm/gemm_perf_testbed.h>
|
||||
#include <tools/test/perf/gemm/gemm_profiler.h>
|
||||
#include <tools/test/perf/gemm/cutlass_dispatch.h>
|
||||
|
||||
namespace perf {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
int profile_igemm(TestbenchOutput &output, TestbenchOptions const &options) {
|
||||
|
||||
typedef perf::GemmProfiler<int8_t, int8_t, int, int, int> GemmProfiler;
|
||||
|
||||
int results = 0;
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::IgemmTraits<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor
|
||||
> GemmTraits;
|
||||
|
||||
typedef typename CutlassDispatchBasic<GemmTraits>::Dispatch Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, GemmProfiler>(output, "igemm_nt", options);
|
||||
}
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::IgemmTraits<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor
|
||||
> GemmTraits;
|
||||
|
||||
typedef typename CutlassDispatchBasic<GemmTraits>::Dispatch Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, GemmProfiler>(output, "igemm_nn", options);
|
||||
}
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::IgemmTraits<
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor
|
||||
> GemmTraits;
|
||||
|
||||
typedef typename CutlassDispatchBasic<GemmTraits>::Dispatch Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, GemmProfiler>(output, "igemm_tn", options);
|
||||
}
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::IgemmTraits<
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor
|
||||
> GemmTraits;
|
||||
|
||||
typedef typename CutlassDispatchBasic<GemmTraits>::Dispatch Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, GemmProfiler>(output, "igemm_tt", options);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace perf
|
||||
@@ -0,0 +1,101 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/sgemm_traits.h>
|
||||
|
||||
#include <tools/test/perf/gemm/gemm_perf_testbed.h>
|
||||
|
||||
#include <tools/test/perf/gemm/gemm_profiler.h>
|
||||
#include <tools/test/perf/gemm/cutlass_dispatch.h>
|
||||
|
||||
namespace perf {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
int profile_sgemm(TestbenchOutput &output, TestbenchOptions const &options) {
|
||||
|
||||
typedef perf::GemmProfiler<float, float, float, float, float> SGemmProfiler;
|
||||
|
||||
int results = 0;
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::SgemmTraits<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<8, 128, 128>
|
||||
> GemmTraits;
|
||||
|
||||
typedef typename CutlassDispatchBasic<GemmTraits>::Dispatch Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, SGemmProfiler>(output, "sgemm_nt", options);
|
||||
}
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::SgemmTraits<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<8, 128, 128>
|
||||
> GemmTraits;
|
||||
|
||||
typedef typename CutlassDispatchBasic<GemmTraits>::Dispatch Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, SGemmProfiler>(output, "sgemm_nn", options);
|
||||
}
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::SgemmTraits<
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<8, 128, 128>
|
||||
> GemmTraits;
|
||||
|
||||
typedef typename CutlassDispatchBasic<GemmTraits>::Dispatch Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, SGemmProfiler>(output, "sgemm_tn", options);
|
||||
}
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::SgemmTraits<
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<8, 128, 128>
|
||||
> GemmTraits;
|
||||
|
||||
typedef typename CutlassDispatchBasic<GemmTraits>::Dispatch Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, SGemmProfiler>(output, "sgemm_tt", options);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace perf
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include <cutlass/wmma_matrix.h>
|
||||
#ifdef CUTLASS_USE_WMMA_API
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
|
||||
#include <tools/test/perf/gemm/gemm_profiler.h>
|
||||
#include <tools/test/perf/gemm/cutlass_dispatch.h>
|
||||
#include <tools/test/perf/gemm/gemm_perf_testbed.h>
|
||||
#include <cutlass/gemm/wmma_gemm_traits.h>
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename Traits>
|
||||
struct WmmaGemmDispatch {
|
||||
|
||||
typedef cutlass::gemm::Gemm<Traits> Gemm;
|
||||
|
||||
typedef typename Gemm::Params Params;
|
||||
|
||||
/// Indicate warp-level GEMM
|
||||
static bool const kThreadMultiplyAdd = false;
|
||||
|
||||
static cutlass::MatrixLayout::Kind const kLayoutA = Traits::kLayoutA;
|
||||
static cutlass::MatrixLayout::Kind const kLayoutB = Traits::kLayoutB;
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// Params argument
|
||||
Params params;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
WmmaGemmDispatch() {}
|
||||
|
||||
/// Initializes params object
|
||||
WmmaGemmDispatch(int m, int n, int k, float alpha, half const* d_a, int lda,
|
||||
half const* d_b, int ldb, float beta, float const* d_c, int ldc,
|
||||
float* d_d, int ldd) {
|
||||
|
||||
params.initialize(m, n, k, alpha, d_a, lda, d_b, ldb, beta, d_c, ldc, d_d, ldd);
|
||||
}
|
||||
|
||||
/// Initializes params object
|
||||
WmmaGemmDispatch(Params const& _params) : params(_params) {}
|
||||
|
||||
/// Launches kernel
|
||||
cudaError_t operator()() { return Gemm::launch(params); }
|
||||
|
||||
/// Determines if problem is aligned (assuming no padding)
|
||||
static bool is_problem_aligned(
|
||||
int m,
|
||||
int n,
|
||||
int k) {
|
||||
|
||||
bool aligned = true;
|
||||
|
||||
if (kLayoutA == cutlass::MatrixLayout::kColumnMajor) {
|
||||
aligned = aligned && !(m % Gemm::Traits::GemmConfig::kScalarsPerLdgA);
|
||||
}
|
||||
else {
|
||||
aligned = aligned && !(k % Gemm::Traits::GemmConfig::kScalarsPerLdgA);
|
||||
}
|
||||
|
||||
if (kLayoutB == cutlass::MatrixLayout::kColumnMajor) {
|
||||
aligned = aligned && !(k % Gemm::Traits::GemmConfig::kScalarsPerLdgB);
|
||||
}
|
||||
else {
|
||||
aligned = aligned && !(n % Gemm::Traits::GemmConfig::kScalarsPerLdgB);
|
||||
}
|
||||
|
||||
aligned = aligned && !(m % Gemm::Traits::GemmConfig::kScalarsPerLdgC);
|
||||
|
||||
return aligned;
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace perf {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
int profile_wmma_gemm(TestbenchOutput &output, TestbenchOptions const &options) {
|
||||
|
||||
typedef perf::GemmProfiler<cutlass::half_t, cutlass::half_t, float, float, float> GemmProfiler;
|
||||
|
||||
int results = 0;
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor>
|
||||
WmmaGemmTraits;
|
||||
|
||||
typedef WmmaGemmDispatch<WmmaGemmTraits> Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, GemmProfiler>(output, "wmma_gemm_nt", options);
|
||||
}
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor>
|
||||
WmmaGemmTraits;
|
||||
|
||||
typedef WmmaGemmDispatch<WmmaGemmTraits> Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, GemmProfiler>(output, "wmma_gemm_nn", options);
|
||||
}
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor>
|
||||
WmmaGemmTraits;
|
||||
|
||||
typedef WmmaGemmDispatch<WmmaGemmTraits> Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, GemmProfiler>(output, "wmma_gemm_tn", options);
|
||||
}
|
||||
|
||||
if (!results) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor>
|
||||
WmmaGemmTraits;
|
||||
|
||||
typedef WmmaGemmDispatch<WmmaGemmTraits> Dispatch;
|
||||
|
||||
profile_gemm<Dispatch, GemmProfiler>(output, "wmma_gemm_tt", options);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace perf
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // defined CUTLASS_USE_WMMA_API
|
||||
@@ -0,0 +1,229 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cutlass/matrix_traits.h>
|
||||
#include <tools/util/command_line.h>
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace perf {
|
||||
|
||||
/// Outcome of test
|
||||
struct Disposition {
|
||||
enum Kind { Unknown = 0, NotRun, Passed, Incorrect, Failed, NotVerified, Invalid };
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace perf
|
||||
|
||||
inline std::ostream &operator<<(std::ostream &out, perf::Disposition::Kind value) {
|
||||
char const *str[] = {
|
||||
"unknown", "not_run", "passed", "incorrect", "failed", "not_verified", "invalid"};
|
||||
if (value >= perf::Disposition::Unknown && value < perf::Disposition::Invalid) {
|
||||
out << str[value];
|
||||
} else {
|
||||
out << str[perf::Disposition::Invalid];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Outputs matrix layout
|
||||
inline std::ostream &operator<<(std::ostream &out, cutlass::MatrixLayout::Kind layout) {
|
||||
out << (layout == cutlass::MatrixLayout::kColumnMajor ? "column" : "row");
|
||||
return out;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace perf {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Size and layout of a GEMM problem
|
||||
struct GemmProblem {
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
int m;
|
||||
int n;
|
||||
int k;
|
||||
cutlass::MatrixLayout::Kind layout_A;
|
||||
cutlass::MatrixLayout::Kind layout_B;
|
||||
|
||||
double alpha;
|
||||
double beta;
|
||||
|
||||
//
|
||||
// Static function members
|
||||
//
|
||||
|
||||
/// Static method to print GemmProblem headers
|
||||
static std::string header() { return "M, N, K, Layout_A, Layout_B, Beta"; }
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
GemmProblem(int _m = 0,
|
||||
int _n = 0,
|
||||
int _k = 0,
|
||||
cutlass::MatrixLayout::Kind _layout_A = cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::Kind _layout_B = cutlass::MatrixLayout::kRowMajor,
|
||||
double _alpha = 1,
|
||||
double _beta = 0)
|
||||
: m(_m), n(_n), k(_k), layout_A(_layout_A), layout_B(_layout_B), alpha(_alpha), beta(_beta) {}
|
||||
|
||||
/// leading dimension of A
|
||||
int lda() const {
|
||||
if (layout_A == cutlass::MatrixLayout::kColumnMajor) {
|
||||
return m;
|
||||
}
|
||||
return k;
|
||||
}
|
||||
|
||||
/// leading dimension of B
|
||||
int ldb() const {
|
||||
if (layout_B == cutlass::MatrixLayout::kColumnMajor) {
|
||||
return k;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/// leading dimension of C
|
||||
int ldc() const { return m; }
|
||||
|
||||
/// Pretty prints output
|
||||
std::ostream &pretty_print(std::ostream &out) const {
|
||||
out << m << "-by-" << n << "-by-" << k << ", A: " << layout_A << "-major, B: " << layout_B
|
||||
<< "-major, beta: " << beta;
|
||||
|
||||
return out;
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace perf
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Prints a problem to an output stream
|
||||
inline std::ostream &operator<<(std::ostream &out, perf::GemmProblem const &problem) {
|
||||
out << problem.m << ", " << problem.n << ", " << problem.k << ", " << problem.layout_A << ", "
|
||||
<< problem.layout_B << ", " << problem.beta;
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace perf {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Result object
|
||||
struct PerformanceResult {
|
||||
|
||||
/// Name of kernel
|
||||
std::string kernel_name;
|
||||
|
||||
/// Problem size
|
||||
GemmProblem problem;
|
||||
|
||||
/// Outcome of test
|
||||
Disposition::Kind disposition;
|
||||
|
||||
/// Runtime in ms
|
||||
double runtime;
|
||||
|
||||
/// Throughput in units of GFLOPs
|
||||
double gflops;
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
PerformanceResult(
|
||||
std::string const &_kernel_name = "",
|
||||
GemmProblem const &_problem = GemmProblem(),
|
||||
Disposition::Kind _disposition = Disposition::NotRun,
|
||||
double _runtime = 0,
|
||||
double _gflops = 0)
|
||||
:
|
||||
kernel_name(_kernel_name),
|
||||
problem(_problem),
|
||||
disposition(_disposition),
|
||||
runtime(_runtime),
|
||||
gflops(_gflops) {}
|
||||
|
||||
/// Displays headers
|
||||
static std::string header() {
|
||||
return std::string("Kernel, ") + GemmProblem::header() +
|
||||
", Disposition, Runtime, GFLOPs";
|
||||
}
|
||||
|
||||
/// Prints human-readable results
|
||||
std::ostream &pretty_print(std::ostream &out) const {
|
||||
|
||||
out << "Kernel: \033[1m" << kernel_name << "\033[0m\n"
|
||||
<< " problem: ";
|
||||
|
||||
std::stringstream disposition_str;
|
||||
if (disposition == Disposition::Passed) {
|
||||
disposition_str << "\033[1m";
|
||||
}
|
||||
else {
|
||||
disposition_str << "\033[1;31m";
|
||||
}
|
||||
disposition_str << disposition << "\033[0m";
|
||||
|
||||
problem.pretty_print(out) << "\n"
|
||||
<< " disposition: " << disposition_str.str() << "\n"
|
||||
<< " runtime: " << runtime << " ms\n\n"
|
||||
<< " performance: \033[1m" << gflops << " GFLOPs\033[0m\n\n";
|
||||
|
||||
return out;
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace perf
|
||||
|
||||
/// Outputs result
|
||||
inline std::ostream &operator<<(std::ostream &out, perf::PerformanceResult const &result) {
|
||||
|
||||
out << result.kernel_name << ", " << result.problem << ", "
|
||||
<< result.disposition << ", " << result.runtime << ", " << result.gflops;
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,583 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <tools/util/command_line.h>
|
||||
|
||||
namespace perf {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Range of problem sizes
|
||||
struct Range {
|
||||
int start;
|
||||
int end;
|
||||
int increment;
|
||||
|
||||
Range(int _start = 0) : start(_start), end(_start), increment(1) {}
|
||||
|
||||
Range(int _start, int _end, int _increment = 1)
|
||||
: start(_start), end(_end), increment(_increment) {}
|
||||
};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Defines a space of problem sizes
|
||||
struct GemmProblemRange {
|
||||
public:
|
||||
/// Range of sizes in GEMM M dimension
|
||||
Range M;
|
||||
|
||||
/// Range of sizes in GEMM N dimension
|
||||
Range N;
|
||||
|
||||
/// Range of sizes in GEMM K dimension
|
||||
Range K;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Constructor to define a space of probelm sizes
|
||||
GemmProblemRange(Range _M = Range(256), Range _N = Range(256), Range _K = Range(256))
|
||||
: M(_M), N(_N), K(_K) {}
|
||||
|
||||
/// Parses a command line argument as a Range object
|
||||
static void get_range(Range &range,
|
||||
cutlass::CommandLine const &args,
|
||||
std::string const &arg,
|
||||
Range const &_default = Range(256)) {
|
||||
range = Range(0, 0, 1);
|
||||
|
||||
if (args.check_cmd_line_flag(arg.c_str())) {
|
||||
std::vector<std::string> values;
|
||||
args.get_cmd_line_arguments(arg.c_str(), values, ':');
|
||||
|
||||
if (values.size() > 0) {
|
||||
std::stringstream ss;
|
||||
ss << values.at(0);
|
||||
ss >> range.start;
|
||||
}
|
||||
|
||||
if (values.size() > 1) {
|
||||
std::stringstream ss;
|
||||
ss << values.at(1);
|
||||
ss >> range.end;
|
||||
} else {
|
||||
range.end = range.start;
|
||||
}
|
||||
|
||||
if (values.size() > 2) {
|
||||
std::stringstream ss;
|
||||
ss << values.at(2);
|
||||
ss >> range.increment;
|
||||
}
|
||||
} else {
|
||||
range = _default;
|
||||
}
|
||||
}
|
||||
|
||||
/// Initializes the GEMM problem size from command line arguments
|
||||
GemmProblemRange(cutlass::CommandLine const &args) {
|
||||
get_range(M, args, "m", Range(10240));
|
||||
get_range(N, args, "n", Range(4096));
|
||||
get_range(K, args, "k", Range(4096));
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Distribution type
|
||||
struct Distribution {
|
||||
/// Variant types
|
||||
enum Kind { Invalid, Uniform, Gaussian, Linear, Identity };
|
||||
|
||||
/// Distribution state
|
||||
union {
|
||||
/// Uniform distribution
|
||||
struct {
|
||||
double min;
|
||||
double max;
|
||||
} uniform;
|
||||
|
||||
/// Gaussian distribution
|
||||
struct {
|
||||
double mean;
|
||||
double stddev;
|
||||
} gaussian;
|
||||
|
||||
/// Elements are linear combination of row and column index
|
||||
struct {
|
||||
double offset;
|
||||
double delta_row;
|
||||
double delta_column;
|
||||
} linear;
|
||||
};
|
||||
|
||||
/// Active variant kind
|
||||
Kind kind;
|
||||
|
||||
/// Random values are cast to integer after scaling by this power of two
|
||||
int int_scale;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
Distribution() : kind(Invalid), int_scale(0) {}
|
||||
|
||||
/// Configures distribution as uniform random
|
||||
Distribution &set_uniform(double _min, double _max, int _int_scale = 0) {
|
||||
kind = Uniform;
|
||||
uniform.min = _min;
|
||||
uniform.max = _max;
|
||||
int_scale = _int_scale;
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Configures distribution as Gaussian distribution
|
||||
Distribution &set_gaussian(double _mean, double _stddev, int _int_scale = 0) {
|
||||
kind = Gaussian;
|
||||
gaussian.mean = _mean;
|
||||
gaussian.stddev = _stddev;
|
||||
int_scale = _int_scale;
|
||||
return *this;
|
||||
}
|
||||
|
||||
|
||||
/// Sets identity
|
||||
Distribution &set_identity() {
|
||||
kind = Identity;
|
||||
return *this;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace perf
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Prints a Distribution to ostream
|
||||
inline std::ostream &operator<<(std::ostream &out, perf::Distribution const &dist) {
|
||||
switch (dist.kind) {
|
||||
case perf::Distribution::Uniform:
|
||||
out << "uniorm, min: " << dist.uniform.min << ", max: " << dist.uniform.max;
|
||||
break;
|
||||
case perf::Distribution::Gaussian:
|
||||
out << "gaussian, mean: " << dist.gaussian.mean << ", stddev: " << dist.gaussian.stddev;
|
||||
break;
|
||||
case perf::Distribution::Linear:
|
||||
out << "linear, mean: " << dist.linear.offset << ", delta_row: " << dist.linear.delta_row
|
||||
<< ", delta_column: " << dist.linear.delta_column;
|
||||
break;
|
||||
case perf::Distribution::Identity:
|
||||
break;
|
||||
default:
|
||||
out << "unknown";
|
||||
}
|
||||
|
||||
out << ", int_scale: " << dist.int_scale;
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace perf {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Defines a vector of string pairs
|
||||
typedef std::vector<std::pair<std::string, std::string> > KeyValueVector;
|
||||
|
||||
/// Defines a const iterator to a KeyValueVector
|
||||
typedef KeyValueVector::const_iterator KeyValueIterator;
|
||||
|
||||
/// Structure captures the initial configuration of matrices
|
||||
struct InitialDistribution {
|
||||
/// Distribution of A matrix operand
|
||||
Distribution dist_A;
|
||||
|
||||
/// Distribution of B matrix operand
|
||||
Distribution dist_B;
|
||||
|
||||
/// Distribution of C matrix operand
|
||||
Distribution dist_C;
|
||||
|
||||
/// Seed for random number generation
|
||||
int64_t seed;
|
||||
|
||||
//
|
||||
// Static function members
|
||||
//
|
||||
|
||||
/// Gets the initial distribution
|
||||
static void get_distribution(cutlass::CommandLine const &args,
|
||||
std::string const &arg,
|
||||
Distribution &dist) {
|
||||
struct {
|
||||
const char *label;
|
||||
Distribution::Kind kind;
|
||||
} distribution_kinds[] = {{"uniform", Distribution::Uniform},
|
||||
{"gaussian", Distribution::Gaussian},
|
||||
{"linear", Distribution::Linear},
|
||||
{"identity", Distribution::Identity},
|
||||
{0, Distribution::Invalid}};
|
||||
|
||||
struct {
|
||||
char const *label;
|
||||
double *member;
|
||||
} members[] = {{"min", &dist.uniform.min},
|
||||
{"max", &dist.uniform.max},
|
||||
{"mean", &dist.gaussian.mean},
|
||||
{"stddev", &dist.gaussian.stddev},
|
||||
{"offset", &dist.linear.offset},
|
||||
{"delta_row", &dist.linear.delta_row},
|
||||
{"delta_column", &dist.linear.delta_column},
|
||||
{0, 0}};
|
||||
|
||||
KeyValueVector values;
|
||||
args.get_cmd_line_argument_pairs(arg.c_str(), values);
|
||||
|
||||
// The parser expects the first token to be a string identifying the distribution type.
|
||||
KeyValueIterator it = values.begin();
|
||||
if (it != values.end()) {
|
||||
for (int i = 0; distribution_kinds[i].label; ++i) {
|
||||
if (it->first == distribution_kinds[i].label) {
|
||||
dist.kind = distribution_kinds[i].kind;
|
||||
break;
|
||||
}
|
||||
}
|
||||
++it;
|
||||
}
|
||||
|
||||
// Subsequent key-value pairs update the named field of the distribution struct.
|
||||
for (; it != values.end(); ++it) {
|
||||
|
||||
// Integer scaling factor - if < 0, no integer rounding is performed.
|
||||
if (it->first == "scale" && !it->second.empty()) {
|
||||
std::stringstream ss;
|
||||
ss << it->second;
|
||||
ss >> dist.int_scale;
|
||||
|
||||
continue; // next token
|
||||
}
|
||||
|
||||
// Casts as integer without scaling
|
||||
if (it->first == "integer") {
|
||||
dist.int_scale = 0;
|
||||
continue; // next token
|
||||
}
|
||||
|
||||
// initialize other members
|
||||
for (int m = 0; members[m].label; ++m) {
|
||||
if (it->first == members[m].label && !it->second.empty()) {
|
||||
std::stringstream ss;
|
||||
ss << it->second;
|
||||
ss >> *(members[m].member);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Basic uniform random distribution
|
||||
InitialDistribution(int64_t _seed = 700) : seed(_seed) {
|
||||
dist_A.set_uniform(-8, 8);
|
||||
dist_B.set_uniform(-8, 8);
|
||||
dist_C.set_uniform(-8, 8);
|
||||
}
|
||||
|
||||
/// Extracts initial distribution from command line arguments
|
||||
InitialDistribution(cutlass::CommandLine const &args) {
|
||||
// Set initial values
|
||||
seed = 700;
|
||||
dist_A.set_uniform(-8, 8);
|
||||
dist_B.set_uniform(-8, 8);
|
||||
dist_C.set_uniform(-8, 8);
|
||||
|
||||
// Update with command line arguments
|
||||
args.get_cmd_line_argument("seed", seed, seed);
|
||||
|
||||
// Update all distributions at once
|
||||
Distribution dist_all;
|
||||
if (args.check_cmd_line_flag("dist")) {
|
||||
get_distribution(args, "dist", dist_all);
|
||||
dist_A = dist_all;
|
||||
dist_B = dist_all;
|
||||
dist_C = dist_all;
|
||||
}
|
||||
|
||||
get_distribution(args, "dist_A", dist_A);
|
||||
get_distribution(args, "dist_B", dist_B);
|
||||
get_distribution(args, "dist_C", dist_C);
|
||||
}
|
||||
};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Defines how to execute the benchmarks
|
||||
struct ExecutionMode {
|
||||
enum Kind {
|
||||
Profile,
|
||||
Verify,
|
||||
Single,
|
||||
Invalid
|
||||
};
|
||||
|
||||
static std::string to_string(Kind kind) {
|
||||
switch (kind) {
|
||||
case Profile: return "profile";
|
||||
case Verify: return "verify";
|
||||
case Single: return "single";
|
||||
default: return "invalid";
|
||||
}
|
||||
}
|
||||
|
||||
static Kind from_string(std::string const &str) {
|
||||
if (str == "profile") return Profile;
|
||||
if (str == "verify") return Verify;
|
||||
if (str == "single") return Single;
|
||||
return Profile;
|
||||
}
|
||||
};
|
||||
|
||||
/// Indicates when the workspace is saved
|
||||
struct WorkspaceSaveMode {
|
||||
enum Kind {
|
||||
Never,
|
||||
Incorrect,
|
||||
Always
|
||||
};
|
||||
|
||||
static std::string to_string(Kind kind) {
|
||||
switch (kind) {
|
||||
case Never: return "never";
|
||||
case Incorrect: return "incorrect";
|
||||
case Always: return "always";
|
||||
default: return "incorrect";
|
||||
}
|
||||
}
|
||||
|
||||
static Kind from_string(std::string const &str) {
|
||||
if (str == "never") return Never;
|
||||
if (str == "incorrect") return Incorrect;
|
||||
if (str == "always") return Always;
|
||||
return Incorrect;
|
||||
}
|
||||
};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Class holding testbench command line options
|
||||
struct TestbenchOptions {
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// Describes the random initial state of the input matrices
|
||||
InitialDistribution initial_distribution;
|
||||
|
||||
// Path to output file name
|
||||
std::string output_filename;
|
||||
|
||||
/// If true, output is appended
|
||||
bool append;
|
||||
|
||||
/// Number of iterations
|
||||
int iterations;
|
||||
|
||||
/// Defines how to run the benchmark
|
||||
ExecutionMode::Kind execution_mode;
|
||||
|
||||
/// Indicates when the workspace is saved
|
||||
WorkspaceSaveMode::Kind save_workspace_mode;
|
||||
|
||||
/// Enabled kernel names
|
||||
std::vector<std::string> kernels;
|
||||
|
||||
/// Scalar value for GEMM
|
||||
double alpha;
|
||||
|
||||
/// Scalar value for GEMM
|
||||
double beta;
|
||||
|
||||
/// Range of problem sizes
|
||||
GemmProblemRange problem_range;
|
||||
|
||||
/// Tags to describe the profiler output
|
||||
KeyValueVector pivot_tags;
|
||||
|
||||
/// If enabled, only the peak performance for a given kernel is reported
|
||||
bool peak_performance;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Constructs the testbench from tags
|
||||
TestbenchOptions(cutlass::CommandLine const &args)
|
||||
: initial_distribution(args),
|
||||
execution_mode(ExecutionMode::Profile),
|
||||
save_workspace_mode(WorkspaceSaveMode::Never),
|
||||
problem_range(args) {
|
||||
|
||||
// fetch command line arguments
|
||||
args.get_cmd_line_argument("iterations", iterations, 25);
|
||||
args.get_cmd_line_argument("append", append, false);
|
||||
args.get_cmd_line_argument("output", output_filename);
|
||||
args.get_cmd_line_argument("alpha", alpha, 1.0);
|
||||
args.get_cmd_line_argument("beta", beta, 0.0);
|
||||
args.get_cmd_line_argument("peak", peak_performance, false);
|
||||
args.get_cmd_line_argument_pairs("tags", pivot_tags);
|
||||
|
||||
if (args.check_cmd_line_flag("execution_mode")) {
|
||||
std::string str;
|
||||
args.get_cmd_line_argument("execution_mode", str);
|
||||
execution_mode = ExecutionMode::from_string(str);
|
||||
}
|
||||
|
||||
if (args.check_cmd_line_flag("save_workspace")) {
|
||||
std::string str;
|
||||
args.get_cmd_line_argument("save_workspace", str);
|
||||
save_workspace_mode = WorkspaceSaveMode::from_string(str);
|
||||
}
|
||||
|
||||
// query for enabled kernels or enable all of them
|
||||
if (args.check_cmd_line_flag("kernels")) {
|
||||
args.get_cmd_line_arguments("kernels", kernels, ',');
|
||||
} else {
|
||||
char const *gemms[] = {"sgemm", "dgemm", "hgemm", "igemm", "wmma_gemm", 0};
|
||||
char const *layouts[] = {"nn", "nt", "tn", "tt", 0};
|
||||
for (int i = 0; gemms[i]; ++i) {
|
||||
for (int j = 0; layouts[j]; ++j) {
|
||||
kernels.push_back(std::string(gemms[i]) + "_" + layouts[j]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if the kernel name appears among the enabled kernels
|
||||
bool kernel_enabled(std::string const &kernel) const {
|
||||
typedef std::vector<std::string>::const_iterator kernel_iterator;
|
||||
|
||||
for (kernel_iterator it = kernels.begin(); it != kernels.end(); ++it) {
|
||||
if (kernel.find(*it) != std::string::npos) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Given the disposition of a GEMM problem, returns true if the results should
|
||||
/// be saved to the file system.
|
||||
bool save_workspace(bool correct) const {
|
||||
if (save_workspace_mode == WorkspaceSaveMode::Always ||
|
||||
(save_workspace_mode == WorkspaceSaveMode::Incorrect && !correct)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Prints the usage statement
|
||||
static void usage(std::ostream &out) {
|
||||
|
||||
out << "cutlass_perf_test [options]\n\n"
|
||||
|
||||
<< " --help\n"
|
||||
|
||||
<< " --append=<true|false*> "
|
||||
<< " If true, appends output to existing CSV file. If false, overwrites.\n"
|
||||
|
||||
<< " --alpha=<alpha> "
|
||||
<< " Value for alpha to be used in GEMM experiments\n"
|
||||
|
||||
<< " --beta=<beta> "
|
||||
<< " Value for beta to be used in GEMM experiments\n"
|
||||
|
||||
<< " --dist_{A,B,C}=<distribution> "
|
||||
<< " Describes the random distribution of each of the input matrix operands.\n"
|
||||
|
||||
<< " --execution_mode=<mode> "
|
||||
<< " Specifies execution mode: profile, verify, single\n"
|
||||
|
||||
<< " --output=<filename.csv> "
|
||||
<< " Writes summary of profiling to specified .csv file\n"
|
||||
|
||||
<< " --iterations=<timing iterations> "
|
||||
<< " maximum number of iterations to execute when profiling\n"
|
||||
|
||||
<< " --m=<height>[:max height[:step]] "
|
||||
<< " Height of GEMM problem (number of rows of C). May specify a range with optional "
|
||||
"step size.\n"
|
||||
|
||||
<< " --n=<width>[:max width[:step]] "
|
||||
<< " Width of GEMM problem (number of columns of C). May specify a range with optional "
|
||||
"step size.\n"
|
||||
|
||||
<< " --k=<depth>[:max depth[:step]] "
|
||||
<< " Size of inner dimension of A and B. May specify a range with optional step size.\n"
|
||||
|
||||
<< " --kernels=<{s|d|h|i|wmma}gemm_{nn,nt,tn,tt}> "
|
||||
<< " Select GEMM datatype and layout to use for tests\n"
|
||||
|
||||
<< " --peak=<bool> "
|
||||
<< " If true, only reports peak performance per kernel after profiling specified "
|
||||
"problem space.\n"
|
||||
|
||||
<< " --save_workspace={*never,incorrect,always} "
|
||||
<< " Specifies when to save the GEMM inputs and results to the filesystem.\n"
|
||||
|
||||
<< " --seed=<seed> "
|
||||
<< " Random seed used by the random number generator in initializing input matrices.\n"
|
||||
|
||||
<< " --tags=<column:tag,...> "
|
||||
<< " Inserts leading columns in output table and uniform values for each column. Useful "
|
||||
"for generating pivot tables.\n"
|
||||
|
||||
<< "\n\n"
|
||||
|
||||
<< "Example usage:\n\n"
|
||||
|
||||
<< "# Runs one problem size for all kernels\n"
|
||||
<< "./tools/test/perf/cutlass_perf_test --m=10240 --n=1024 --k=1024\n\n"
|
||||
|
||||
<< "# Varies GEMM K dimension for SGEMM and IGEMM with column-major multiplicands\n"
|
||||
<< "./tools/test/perf/cutlass_perf_test --m=10240 --n=4096 --k=1024:8192:128 "
|
||||
"--kernels=sgemm_nn,igemm_nn\n\n"
|
||||
|
||||
<< std::flush;
|
||||
}
|
||||
};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace perf
|
||||
@@ -0,0 +1,159 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <fstream>
|
||||
|
||||
#include <tools/test/perf/performance_result.h>
|
||||
#include <tools/test/perf/testbench_options.h>
|
||||
#include <tools/util/command_line.h>
|
||||
|
||||
namespace perf {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Wraps an output stream and constructs a comma-separated value table of results
|
||||
class TestbenchOutput {
|
||||
public:
|
||||
/// Options to test environment
|
||||
TestbenchOptions const &options;
|
||||
|
||||
/// Possibly open output file name
|
||||
std::ofstream output_file;
|
||||
|
||||
/// Pointer to either &std::cout or output_file
|
||||
std::ostream *output_ptr;
|
||||
|
||||
/// if true, output is also printed to std::cout in human readable form
|
||||
bool buffer_csv_output;
|
||||
|
||||
/// Vector holding performance results
|
||||
std::vector<PerformanceResult> buffered_perf_results;
|
||||
|
||||
private:
|
||||
/// Opens the output file and updates output_ptr
|
||||
void initialize_output_file() {
|
||||
std::ifstream test_file(options.output_filename.c_str());
|
||||
if (options.append && test_file.good()) {
|
||||
output_file.open(options.output_filename.c_str(), std::ios::app);
|
||||
} else {
|
||||
output_file.open(options.output_filename.c_str());
|
||||
output_file << header() << std::endl;
|
||||
}
|
||||
output_ptr = &output_file;
|
||||
}
|
||||
|
||||
public:
|
||||
/// Emits the header to the output table
|
||||
std::string header() {
|
||||
std::stringstream ss;
|
||||
|
||||
// pivot tags
|
||||
for (KeyValueIterator tag_it = options.pivot_tags.begin(); tag_it != options.pivot_tags.end();
|
||||
++tag_it) {
|
||||
ss << tag_it->first << ", ";
|
||||
}
|
||||
|
||||
// performance result header
|
||||
ss << PerformanceResult::header();
|
||||
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
/// Constructs a TestbenchoutOutput object from command line options
|
||||
TestbenchOutput(TestbenchOptions const &_options) : options(_options), buffer_csv_output(true) {
|
||||
if (!options.output_filename.empty()) {
|
||||
initialize_output_file();
|
||||
buffer_csv_output = false;
|
||||
} else {
|
||||
output_ptr = &std::cout;
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes output to CSV
|
||||
~TestbenchOutput() {
|
||||
std::cout << std::endl;
|
||||
if (buffer_csv_output) {
|
||||
out() << "\n\n" << header() << std::endl;
|
||||
for (std::vector<PerformanceResult>::const_iterator it = buffered_perf_results.begin();
|
||||
it != buffered_perf_results.end();
|
||||
++it) {
|
||||
write_csv(*it);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a reference to an std::ostream instance for writing
|
||||
std::ostream &out() { return *output_ptr; }
|
||||
|
||||
/// Called to indicate a new problem will be output
|
||||
TestbenchOutput &begin_problem() {
|
||||
std::cout << "\n============================================================================\n";
|
||||
|
||||
for (KeyValueIterator tag_it = options.pivot_tags.begin(); tag_it != options.pivot_tags.end();
|
||||
++tag_it) {
|
||||
std::cout << tag_it->first << ": " << tag_it->second << std::endl;
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Writes a performance result to CSV output
|
||||
TestbenchOutput &write_csv(PerformanceResult const &result) {
|
||||
// pivot tags
|
||||
for (KeyValueIterator tag_it = options.pivot_tags.begin(); tag_it != options.pivot_tags.end();
|
||||
++tag_it) {
|
||||
out() << tag_it->second << ", ";
|
||||
}
|
||||
|
||||
out() << result << std::endl;
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Prints the output without appending it for CSV writing
|
||||
TestbenchOutput &pretty_print(PerformanceResult const &result) {
|
||||
result.pretty_print(std::cout) << std::endl;
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Emits the result as output
|
||||
TestbenchOutput &append(PerformanceResult const &result) {
|
||||
if (buffer_csv_output) {
|
||||
buffered_perf_results.push_back(result);
|
||||
} else {
|
||||
write_csv(result);
|
||||
}
|
||||
|
||||
pretty_print(result);
|
||||
|
||||
return *this;
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace perf
|
||||
Reference in New Issue
Block a user