CUTLASS 2.4 (Implicit GEMM convolution) (#147)
CUTLASS 2.4 (Implicit GEMM Convolution) Co-authored-by: Manish Gupta <manigupta@nvidia.com>, Haicheng Wu <haichengw@nvidia.com>, Dustyn Blasig <dblasig@nvidia.com>, Andrew Kerr <akerr@nvidia.com>
This commit is contained in:
Manish Gupta
co-authored by
Manish Gupta <manigupta@nvidia.com>, Haicheng Wu <haichengw@nvidia.com>, Dustyn Blasig <dblasig@nvidia.com>, Andrew Kerr <akerr@nvidia.com>
Manish Gupta <manigupta@nvidia.com>, Haicheng Wu <haichengw@nvidia.com>, Dustyn Blasig <dblasig@nvidia.com>, Andrew Kerr <akerr@nvidia.com>
parent
c2b80ad4e4
commit
6615010cd0
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,431 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Defines profiling functionality for convolution
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <algorithm>
|
||||
#include <unordered_map>
|
||||
|
||||
// CUTLASS Library includes
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/util.h"
|
||||
#include "cutlass/library/handle.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
#include "cutlass/library/singleton.h"
|
||||
|
||||
// Profiler includes
|
||||
#include "options.h"
|
||||
#include "device_context.h"
|
||||
#include "operation_profiler.h"
|
||||
#include "performance_result.h"
|
||||
#include "problem_space.h"
|
||||
#include "reduction_operation_profiler.h"
|
||||
#if CUTLASS_ENABLE_CUDNN
|
||||
#include "cudnn_helpers.h"
|
||||
#endif //#if CUTLASS_ENABLE_CUDNN
|
||||
#include "debug.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Abstract base class for each math function
|
||||
class Conv2dOperationProfiler : public OperationProfiler {
|
||||
public:
|
||||
|
||||
/// Problem structure obtained from problem space
|
||||
struct Conv2dProblem {
|
||||
|
||||
int64_t n, h, w, c, p, q, k, r, s;
|
||||
int64_t pad_h, pad_w;
|
||||
int64_t stride_h, stride_w;
|
||||
int64_t dilation_h, dilation_w;
|
||||
|
||||
std::vector<uint8_t> alpha;
|
||||
std::vector<uint8_t> beta;
|
||||
|
||||
library::SplitKMode split_k_mode;
|
||||
int64_t split_k_slices;
|
||||
|
||||
library::ConvModeID conv_mode;
|
||||
|
||||
library::Provider eq_gemm_provider;
|
||||
|
||||
// convolution with parallel interleaved reduction
|
||||
// convolution epilogue (alpha, beta) = (1.0, 0.0)
|
||||
// reduction epilogue (alpha, beta) = (Conv2dProblem::alpha, Conv2dProblem::beta)
|
||||
std::vector<uint8_t> alpha_one;
|
||||
std::vector<uint8_t> beta_zero;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Total number of bytes loaded
|
||||
int64_t bytes(library::ConvDescription const &operation_desc) const;
|
||||
|
||||
/// Total number of flops computed
|
||||
int64_t flops(library::ConvDescription const &operation_desc) const;
|
||||
|
||||
void set_default_output_size() {
|
||||
p = ((h + pad_h - r * dilation_h) / stride_h) + 1;
|
||||
q = ((w + pad_w - s * dilation_w) / stride_w) + 1;
|
||||
}
|
||||
|
||||
// Returns equivalent gemm problem size for convolution
|
||||
cutlass::gemm::GemmCoord eq_gemm_size(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return cutlass::gemm::GemmCoord(int(n * p * q), int(k), int(r * s * c));
|
||||
case library::ConvKind::kDgrad: return cutlass::gemm::GemmCoord(int(n * h * w), int(c), int(k * r * s));
|
||||
case library::ConvKind::kWgrad: return cutlass::gemm::GemmCoord(int(k), int(r * s * c), int(n * p * q));
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns extent for tensor A
|
||||
std::vector<int> extent_a(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return {int(n), int(h), int(w), int(c)};
|
||||
case library::ConvKind::kDgrad: return {int(n), int(p), int(q), int(k)};
|
||||
case library::ConvKind::kWgrad: return {int(n), int(p), int(q), int(k)};
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns extent for tensor B
|
||||
std::vector<int> extent_b(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return {int(k), int(r), int(s), int(c)};
|
||||
case library::ConvKind::kDgrad: return {int(k), int(r), int(s), int(c)};
|
||||
case library::ConvKind::kWgrad: return {int(n), int(h), int(w), int(c)};
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns extent for tensor C
|
||||
std::vector<int> extent_c(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return {int(n), int(p), int(q), int(k)};
|
||||
case library::ConvKind::kDgrad: return {int(n), int(h), int(w), int(c)};
|
||||
case library::ConvKind::kWgrad: return {int(k), int(r), int(s), int(c)};
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns layout for equivalent gemm matrix A
|
||||
library::LayoutTypeID eq_gemm_layout_a(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return library::LayoutTypeID::kRowMajor; // TN Gemm
|
||||
case library::ConvKind::kDgrad: return library::LayoutTypeID::kRowMajor; // TT Gemm
|
||||
case library::ConvKind::kWgrad: return library::LayoutTypeID::kColumnMajor; // NT Gemm
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns layout for equivalent gemm matrix B
|
||||
library::LayoutTypeID eq_gemm_layout_b(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return library::LayoutTypeID::kColumnMajor; // TN Gemm
|
||||
case library::ConvKind::kDgrad: return library::LayoutTypeID::kRowMajor; // TT Gemm
|
||||
case library::ConvKind::kWgrad: return library::LayoutTypeID::kRowMajor; // NT Gemm
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns layout for equivalent gemm matrix C
|
||||
library::LayoutTypeID eq_gemm_layout_c(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
// Gemm operator assumes column-major output
|
||||
case library::ConvKind::kFprop:
|
||||
case library::ConvKind::kDgrad:
|
||||
case library::ConvKind::kWgrad: return library::LayoutTypeID::kColumnMajor;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns leading dimenstion for equivalent gemm matrix A
|
||||
int64_t eq_gemm_lda(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return eq_gemm_size(conv_kind).k();
|
||||
case library::ConvKind::kDgrad: return eq_gemm_size(conv_kind).k();
|
||||
case library::ConvKind::kWgrad: return eq_gemm_size(conv_kind).m();
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns leading dimenstion for equivalent gemm matrix B
|
||||
int64_t eq_gemm_ldb(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return eq_gemm_size(conv_kind).k();
|
||||
case library::ConvKind::kDgrad: return eq_gemm_size(conv_kind).n();
|
||||
case library::ConvKind::kWgrad: return eq_gemm_size(conv_kind).n();
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns leading dimenstion for equivalent gemm matrix C
|
||||
int64_t eq_gemm_ldc(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop:
|
||||
case library::ConvKind::kDgrad:
|
||||
case library::ConvKind::kWgrad: return eq_gemm_size(conv_kind).m();
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/// Workspace used
|
||||
struct Conv2dWorkspace {
|
||||
|
||||
/// Conv device allocations
|
||||
DeviceAllocation *A;
|
||||
DeviceAllocation *B;
|
||||
DeviceAllocation *C;
|
||||
DeviceAllocation *Computed;
|
||||
DeviceAllocation *Reference;
|
||||
|
||||
/// Library configuration and arguments for convolution operator
|
||||
library::Conv2dConfiguration configuration;
|
||||
library::ConvArguments arguments;
|
||||
|
||||
/// Number of copies of the problem workspace which are visited sequentially during
|
||||
/// profiling to avoid camping in the last level cache.
|
||||
int problem_count;
|
||||
|
||||
/// Buffer used for the cutlass conv2d operations' host workspace
|
||||
std::vector<uint8_t> host_workspace;
|
||||
|
||||
/// Buffer used for the cutlass operations' device workspace
|
||||
DeviceAllocation device_workspace;
|
||||
|
||||
/// Library configuration and arguments for reduction operator
|
||||
library::ReductionConfiguration reduction_configuration;
|
||||
library::ReductionArguments reduction_arguments;
|
||||
|
||||
/// Buffer used for the cutlass reduction operations' host workspace
|
||||
std::vector<uint8_t> reduction_host_workspace;
|
||||
|
||||
/// Host data buffers for host reference operation
|
||||
/// host buffer for tensor
|
||||
std::vector<uint8_t> host_tensor_a;
|
||||
|
||||
/// host buffer for tensor b
|
||||
std::vector<uint8_t> host_tensor_b;
|
||||
|
||||
/// host buffer for tensor c
|
||||
std::vector<uint8_t> host_tensor_c;
|
||||
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
Conv2dWorkspace():
|
||||
A(nullptr), B(nullptr), C(nullptr), Computed(nullptr), Reference(nullptr) { }
|
||||
|
||||
// Returns stride vector for tensor A
|
||||
std::vector<int> stride_a(library::ConvKind const &conv_kind) {
|
||||
return {
|
||||
configuration.layout_a(conv_kind).stride()[0],
|
||||
configuration.layout_a(conv_kind).stride()[1],
|
||||
configuration.layout_a(conv_kind).stride()[2]
|
||||
};
|
||||
}
|
||||
|
||||
// Returns stride vector for tensor B
|
||||
std::vector<int> stride_b(library::ConvKind const &conv_kind) {
|
||||
|
||||
return {
|
||||
configuration.layout_b(conv_kind).stride()[0],
|
||||
configuration.layout_b(conv_kind).stride()[1],
|
||||
configuration.layout_b(conv_kind).stride()[2]
|
||||
};
|
||||
}
|
||||
|
||||
// Returns stride vector for tensor C
|
||||
std::vector<int> stride_c(library::ConvKind const &conv_kind) {
|
||||
|
||||
return {
|
||||
configuration.layout_c(conv_kind).stride()[0],
|
||||
configuration.layout_c(conv_kind).stride()[1],
|
||||
configuration.layout_c(conv_kind).stride()[2]
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
protected:
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// CONV problem obtained from problem space
|
||||
Conv2dProblem problem_;
|
||||
|
||||
/// Device memory allocations
|
||||
Conv2dWorkspace conv_workspace_;
|
||||
|
||||
/// CUTLASS parallel reduction operation to follow this* conv2d operation
|
||||
library::Operation const *reduction_op_;
|
||||
|
||||
public:
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Ctor
|
||||
Conv2dOperationProfiler(Options const &options);
|
||||
|
||||
/// Destructor
|
||||
virtual ~Conv2dOperationProfiler();
|
||||
|
||||
/// Prints usage statement for the math function
|
||||
virtual void print_usage(std::ostream &out) const;
|
||||
|
||||
/// Prints examples
|
||||
virtual void print_examples(std::ostream &out) const;
|
||||
|
||||
/// Extracts the problem dimensions
|
||||
virtual Status initialize_configuration(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Initializes workspace
|
||||
virtual Status initialize_workspace(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
virtual bool verify_cutlass(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Measures performance results
|
||||
virtual bool profile(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
protected:
|
||||
/// Method to profile an initialized CUTLASS operation
|
||||
virtual Status profile_cutlass_(
|
||||
double &runtime,
|
||||
Options const &options,
|
||||
library::Operation const *operation,
|
||||
void *arguments,
|
||||
void *host_workspace,
|
||||
void *device_workspace);
|
||||
|
||||
|
||||
/// Initialize reduction problem dimenstions and library::Operation
|
||||
bool initialize_reduction_configuration_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Initializes the performance result
|
||||
void initialize_result_(
|
||||
PerformanceResult &result,
|
||||
Options const &options,
|
||||
library::ConvDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space);
|
||||
|
||||
/// Verifies CUTLASS against host reference
|
||||
bool verify_with_host_reference_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Verifies CUTLASS against device reference
|
||||
bool verify_with_device_reference_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
#if CUTLASS_ENABLE_CUDNN
|
||||
|
||||
/// Verifies CUTLASS against cudnn reference
|
||||
bool verify_with_cudnn_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
#endif //#if CUTLASS_ENABLE_CUDNN
|
||||
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,441 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Defines profiling functionality for convolution
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <algorithm>
|
||||
#include <unordered_map>
|
||||
|
||||
// CUTLASS Library includes
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/util.h"
|
||||
#include "cutlass/library/handle.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
#include "cutlass/library/singleton.h"
|
||||
|
||||
// Profiler includes
|
||||
#include "options.h"
|
||||
#include "device_context.h"
|
||||
#include "operation_profiler.h"
|
||||
#include "performance_result.h"
|
||||
#include "problem_space.h"
|
||||
#include "reduction_operation_profiler.h"
|
||||
#if CUTLASS_ENABLE_CUDNN
|
||||
#include "cudnn_helpers.h"
|
||||
#endif //#if CUTLASS_ENABLE_CUDNN
|
||||
#include "debug.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Abstract base class for each math function
|
||||
class Conv3dOperationProfiler : public OperationProfiler {
|
||||
public:
|
||||
|
||||
/// Problem structure obtained from problem space
|
||||
struct Conv3dProblem {
|
||||
|
||||
int64_t n, d, h, w, c, z, p, q, k, t, r, s;
|
||||
int64_t pad_d, pad_h, pad_w;
|
||||
int64_t stride_d, stride_h, stride_w;
|
||||
int64_t dilation_d, dilation_h, dilation_w;
|
||||
|
||||
std::vector<uint8_t> alpha;
|
||||
std::vector<uint8_t> beta;
|
||||
|
||||
library::SplitKMode split_k_mode;
|
||||
int64_t split_k_slices;
|
||||
|
||||
library::ConvModeID conv_mode;
|
||||
|
||||
library::Provider eq_gemm_provider;
|
||||
|
||||
// convolution with parallel interleaved reduction
|
||||
// convolution epilogue (alpha, beta) = (1.0, 0.0)
|
||||
// reduction epilogue (alpha, beta) = (Conv3dProblem::alpha, Conv3dProblem::beta)
|
||||
std::vector<uint8_t> alpha_one;
|
||||
std::vector<uint8_t> beta_zero;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Total number of bytes loaded
|
||||
int64_t bytes(library::ConvDescription const &operation_desc) const;
|
||||
|
||||
/// Total number of flops computed
|
||||
int64_t flops(library::ConvDescription const &operation_desc) const;
|
||||
|
||||
/// Infers output size from theinput size, padding, stride, and dilation
|
||||
void set_default_output_size() {
|
||||
z = ((d + pad_d - t * dilation_d) / stride_d) + 1;
|
||||
p = ((h + pad_h - r * dilation_h) / stride_h) + 1;
|
||||
q = ((w + pad_w - s * dilation_w) / stride_w) + 1;
|
||||
}
|
||||
|
||||
// Returns equivalent gemm problem size for convolution
|
||||
cutlass::gemm::GemmCoord eq_gemm_size(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return cutlass::gemm::GemmCoord(int(n * z * p * q), int(k), int(t * r * s * c));
|
||||
case library::ConvKind::kDgrad: return cutlass::gemm::GemmCoord(int(n * d * h * w), int(c), int(t * r * s * k));
|
||||
case library::ConvKind::kWgrad: return cutlass::gemm::GemmCoord(int(k), int(t * r * s * c), int(n * z * p * q));
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns extent for tensor A
|
||||
std::vector<int> extent_a(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return {int(n), int(d), int(h), int(w), int(c)};
|
||||
case library::ConvKind::kDgrad: return {int(n), int(z), int(p), int(q), int(k)};
|
||||
case library::ConvKind::kWgrad: return {int(n), int(z), int(p), int(q), int(k)};
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns extent for tensor B
|
||||
std::vector<int> extent_b(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return {int(k), int(t), int(r), int(s), int(c)};
|
||||
case library::ConvKind::kDgrad: return {int(k), int(t), int(r), int(s), int(c)};
|
||||
case library::ConvKind::kWgrad: return {int(n), int(d), int(h), int(w), int(c)};
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns extent for tensor C
|
||||
std::vector<int> extent_c(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return {int(n), int(z), int(p), int(q), int(k)};
|
||||
case library::ConvKind::kDgrad: return {int(n), int(d), int(h), int(w), int(c)};
|
||||
case library::ConvKind::kWgrad: return {int(k), int(t), int(r), int(s), int(c)};
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns layout for equivalent gemm matrix A
|
||||
library::LayoutTypeID eq_gemm_layout_a(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return library::LayoutTypeID::kRowMajor; // TN Gemm
|
||||
case library::ConvKind::kDgrad: return library::LayoutTypeID::kRowMajor; // TT Gemm
|
||||
case library::ConvKind::kWgrad: return library::LayoutTypeID::kColumnMajor; // NT Gemm
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns layout for equivalent gemm matrix B
|
||||
library::LayoutTypeID eq_gemm_layout_b(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return library::LayoutTypeID::kColumnMajor; // TN Gemm
|
||||
case library::ConvKind::kDgrad: return library::LayoutTypeID::kRowMajor; // TT Gemm
|
||||
case library::ConvKind::kWgrad: return library::LayoutTypeID::kRowMajor; // NT Gemm
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns layout for equivalent gemm matrix C
|
||||
library::LayoutTypeID eq_gemm_layout_c(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
// Gemm operator assumes column-major output
|
||||
case library::ConvKind::kFprop:
|
||||
case library::ConvKind::kDgrad:
|
||||
case library::ConvKind::kWgrad: return library::LayoutTypeID::kColumnMajor;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns leading dimenstion for equivalent gemm matrix A
|
||||
int64_t eq_gemm_lda(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return eq_gemm_size(conv_kind).k();
|
||||
case library::ConvKind::kDgrad: return eq_gemm_size(conv_kind).k();
|
||||
case library::ConvKind::kWgrad: return eq_gemm_size(conv_kind).m();
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns leading dimenstion for equivalent gemm matrix B
|
||||
int64_t eq_gemm_ldb(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return eq_gemm_size(conv_kind).k();
|
||||
case library::ConvKind::kDgrad: return eq_gemm_size(conv_kind).n();
|
||||
case library::ConvKind::kWgrad: return eq_gemm_size(conv_kind).n();
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns leading dimenstion for equivalent gemm matrix C
|
||||
int64_t eq_gemm_ldc(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop:
|
||||
case library::ConvKind::kDgrad:
|
||||
case library::ConvKind::kWgrad: return eq_gemm_size(conv_kind).m();
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/// Workspace used
|
||||
struct Conv2dWorkspace {
|
||||
|
||||
/// Conv device allocations
|
||||
DeviceAllocation *A;
|
||||
DeviceAllocation *B;
|
||||
DeviceAllocation *C;
|
||||
DeviceAllocation *Computed;
|
||||
DeviceAllocation *Reference;
|
||||
|
||||
/// Library configuration and arguments for convolution operator
|
||||
library::Conv3dConfiguration configuration;
|
||||
library::ConvArguments arguments;
|
||||
|
||||
/// Number of copies of the problem workspace which are visited sequentially during
|
||||
/// profiling to avoid camping in the last level cache.
|
||||
int problem_count;
|
||||
|
||||
/// Buffer used for the cutlass conv2d operations' host workspace
|
||||
std::vector<uint8_t> host_workspace;
|
||||
|
||||
/// Buffer used for the cutlass operations' device workspace
|
||||
DeviceAllocation device_workspace;
|
||||
|
||||
/// Library configuration and arguments for reduction operator
|
||||
library::ReductionConfiguration reduction_configuration;
|
||||
library::ReductionArguments reduction_arguments;
|
||||
|
||||
/// Buffer used for the cutlass reduction operations' host workspace
|
||||
std::vector<uint8_t> reduction_host_workspace;
|
||||
|
||||
/// Host data buffers for host reference operation
|
||||
/// host buffer for tensor
|
||||
std::vector<uint8_t> host_tensor_a;
|
||||
|
||||
/// host buffer for tensor b
|
||||
std::vector<uint8_t> host_tensor_b;
|
||||
|
||||
/// host buffer for tensor c
|
||||
std::vector<uint8_t> host_tensor_c;
|
||||
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
Conv2dWorkspace():
|
||||
A(nullptr), B(nullptr), C(nullptr), Computed(nullptr), Reference(nullptr) { }
|
||||
|
||||
// Returns stride vector for tensor A
|
||||
std::vector<int> stride_a(library::ConvKind const &conv_kind) {
|
||||
return {
|
||||
configuration.layout_a(conv_kind).stride()[0],
|
||||
configuration.layout_a(conv_kind).stride()[1],
|
||||
configuration.layout_a(conv_kind).stride()[2],
|
||||
configuration.layout_a(conv_kind).stride()[3]
|
||||
};
|
||||
}
|
||||
|
||||
// Returns stride vector for tensor B
|
||||
std::vector<int> stride_b(library::ConvKind const &conv_kind) {
|
||||
|
||||
return {
|
||||
configuration.layout_b(conv_kind).stride()[0],
|
||||
configuration.layout_b(conv_kind).stride()[1],
|
||||
configuration.layout_b(conv_kind).stride()[2],
|
||||
configuration.layout_b(conv_kind).stride()[3]
|
||||
};
|
||||
}
|
||||
|
||||
// Returns stride vector for tensor C
|
||||
std::vector<int> stride_c(library::ConvKind const &conv_kind) {
|
||||
|
||||
return {
|
||||
configuration.layout_c(conv_kind).stride()[0],
|
||||
configuration.layout_c(conv_kind).stride()[1],
|
||||
configuration.layout_c(conv_kind).stride()[2],
|
||||
configuration.layout_c(conv_kind).stride()[3]
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
protected:
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// CONV problem obtained from problem space
|
||||
Conv3dProblem problem_;
|
||||
|
||||
/// Device memory allocations
|
||||
Conv2dWorkspace conv_workspace_;
|
||||
|
||||
/// CUTLASS parallel reduction operation to follow this* conv2d operation
|
||||
library::Operation const *reduction_op_;
|
||||
|
||||
public:
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Ctor
|
||||
Conv3dOperationProfiler(Options const &options);
|
||||
|
||||
/// Destructor
|
||||
virtual ~Conv3dOperationProfiler();
|
||||
|
||||
/// Prints usage statement for the math function
|
||||
virtual void print_usage(std::ostream &out) const;
|
||||
|
||||
/// Prints examples
|
||||
virtual void print_examples(std::ostream &out) const;
|
||||
|
||||
/// Extracts the problem dimensions
|
||||
virtual Status initialize_configuration(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Initializes workspace
|
||||
virtual Status initialize_workspace(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
virtual bool verify_cutlass(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Measures performance results
|
||||
virtual bool profile(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
protected:
|
||||
|
||||
/// Updates the arguments structure for the CUTLASS operator based on
|
||||
/// the problem index.
|
||||
void set_cutlass_operator_arguments_(int problem_idx = 0);
|
||||
|
||||
/// Method to profile an initialized CUTLASS operation
|
||||
virtual Status profile_cutlass_(
|
||||
double &runtime,
|
||||
Options const &options,
|
||||
library::Operation const *operation,
|
||||
void *arguments,
|
||||
void *host_workspace,
|
||||
void *device_workspace);
|
||||
|
||||
/// Initialize reduction problem dimenstions and library::Operation
|
||||
bool initialize_reduction_configuration_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Initializes the performance result
|
||||
void initialize_result_(
|
||||
PerformanceResult &result,
|
||||
Options const &options,
|
||||
library::ConvDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space);
|
||||
|
||||
/// Verifies CUTLASS against host reference
|
||||
bool verify_with_host_reference_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Verifies CUTLASS against device reference
|
||||
bool verify_with_device_reference_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
#if CUTLASS_ENABLE_CUDNN
|
||||
|
||||
/// Verifies CUTLASS against cudnn reference
|
||||
bool verify_with_cudnn_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
#endif //#if CUTLASS_ENABLE_CUDNN
|
||||
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,485 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Helper functions for mapping CUTLASS concepts to cuDNN.
|
||||
*/
|
||||
#if CUTLASS_ENABLE_CUDNN
|
||||
|
||||
#include <stdexcept>
|
||||
|
||||
#include "cudnn_helpers.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/// Converts a cuDNN status to cutlass::Status
|
||||
Status get_cutlass_status(cudnnStatus_t cudnn_status) {
|
||||
|
||||
if (cudnn_status == CUDNN_STATUS_SUCCESS) {
|
||||
return Status::kSuccess;
|
||||
}
|
||||
else if (cudnn_status == CUDNN_STATUS_INVALID_VALUE) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
if (cudnn_status == CUDNN_STATUS_NOT_SUPPORTED) {
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
return Status::kErrorInternal;
|
||||
}
|
||||
|
||||
/// Converts a cuDNN status to cutlass::profiler::Disposition
|
||||
Disposition get_cutlass_disposition(cudnnStatus_t cudnn_status) {
|
||||
|
||||
if (cudnn_status == CUDNN_STATUS_INVALID_VALUE) {
|
||||
return Disposition::kInvalidProblem;
|
||||
}
|
||||
else if (cudnn_status == CUDNN_STATUS_NOT_SUPPORTED) {
|
||||
return Disposition::kNotSupported;
|
||||
}
|
||||
return Disposition::kFailed;
|
||||
}
|
||||
|
||||
/// Checks cudnnStatus_t converts to cutlas status and returns if Status::kSuccess o.w. throws exception
|
||||
Status checkCudnnErr(cudnnStatus_t cudnn_status) {
|
||||
Status cutlass_status = get_cutlass_status(cudnn_status);
|
||||
if(cutlass_status != Status::kSuccess) {
|
||||
throw std::runtime_error("checkCudnnErr failed");
|
||||
}
|
||||
return cutlass_status;
|
||||
}
|
||||
|
||||
/// Maps a CUTLASS conv mode to a cuDNN cudnnConvolutionMode_t
|
||||
bool get_cudnn_conv_mode(cudnnConvolutionMode_t &cudnn_conv_mode, conv::Mode conv_mode) {
|
||||
switch (conv_mode) {
|
||||
case conv::Mode::kCrossCorrelation:
|
||||
cudnn_conv_mode = CUDNN_CROSS_CORRELATION;
|
||||
return true;
|
||||
case conv::Mode::kConvolution:
|
||||
cudnn_conv_mode = CUDNN_CONVOLUTION;
|
||||
return true;
|
||||
default: break;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Maps a CUTLASS tensor layout to a cuDNN cudnnTensorFormat_t
|
||||
bool get_cudnn_layout(cudnnTensorFormat_t &cudnn_layout, library::LayoutTypeID layout) {
|
||||
switch (layout) {
|
||||
// cudnn uses the same enum for TensorNC*HW along nDim (ConvDescription::conv_dim)
|
||||
case library::LayoutTypeID::kTensorNCHW:
|
||||
case library::LayoutTypeID::kTensorNCDHW:
|
||||
cudnn_layout = CUDNN_TENSOR_NCHW;
|
||||
return true;
|
||||
case library::LayoutTypeID::kTensorNHWC:
|
||||
case library::LayoutTypeID::kTensorNDHWC:
|
||||
cudnn_layout = CUDNN_TENSOR_NHWC;
|
||||
return true;
|
||||
default: break;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Maps a CUTLASS numeric type to a cuDNN cudnnDataType_t
|
||||
bool get_cudnn_datatype(cudnnDataType_t &cudnn_element_type, library::NumericTypeID element_type) {
|
||||
switch (element_type) {
|
||||
case library::NumericTypeID::kF16:
|
||||
cudnn_element_type = CUDNN_DATA_HALF;
|
||||
return true;
|
||||
|
||||
case library::NumericTypeID::kF32:
|
||||
cudnn_element_type = CUDNN_DATA_FLOAT;
|
||||
return true;
|
||||
|
||||
case library::NumericTypeID::kF64:
|
||||
cudnn_element_type = CUDNN_DATA_DOUBLE;
|
||||
return true;
|
||||
|
||||
case library::NumericTypeID::kS2:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kS4:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kS8:
|
||||
cudnn_element_type = CUDNN_DATA_INT8;
|
||||
return true;
|
||||
|
||||
case library::NumericTypeID::kS16:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kS32:
|
||||
cudnn_element_type = CUDNN_DATA_INT32;
|
||||
return true;
|
||||
|
||||
case library::NumericTypeID::kS64:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kU2:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kU4:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kU8:
|
||||
cudnn_element_type = CUDNN_DATA_UINT8;
|
||||
return true;
|
||||
|
||||
case library::NumericTypeID::kU16:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kU32:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kU64:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kB1:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kInvalid:
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Maps CUTLASS math OpcodeClassID and MathOperationID to cuDNN math_type
|
||||
bool get_cudnn_mathtype(cudnnMathType_t &cudnn_math_type, library::ConvDescription const &conv_desc) {
|
||||
|
||||
switch (conv_desc.tile_description.math_instruction.opcode_class) {
|
||||
|
||||
case library::OpcodeClassID::kTensorOp:
|
||||
{
|
||||
cudnn_math_type = CUDNN_TENSOR_OP_MATH;
|
||||
|
||||
library::MathOperationID math_op = conv_desc.tile_description.math_instruction.math_operation;
|
||||
|
||||
// Allow conversion on input data type for fast math operations
|
||||
if (math_op == library::MathOperationID::kMultiplyAddFastF16 ||
|
||||
math_op == library::MathOperationID::kMultiplyAddFastBF16)
|
||||
{
|
||||
cudnn_math_type = CUDNN_TENSOR_OP_MATH_ALLOW_CONVERSION;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
case library::OpcodeClassID::kSimt:
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Cudnn compute type seems to be hardcoded to float (To handle a possible cudnn issue)
|
||||
float cast_cudnn_compute_type_to_float(library::NumericTypeID type, void const * src) {
|
||||
|
||||
switch (type) {
|
||||
case library::NumericTypeID::kF16:
|
||||
{
|
||||
return float(*(static_cast<half_t const*>(src)));
|
||||
}
|
||||
case library::NumericTypeID::kF32:
|
||||
{
|
||||
return float(*(static_cast<float const*>(src)));
|
||||
}
|
||||
case library::NumericTypeID::kS32:
|
||||
{
|
||||
return float(*(static_cast<int const*>(src)));
|
||||
}
|
||||
default:
|
||||
throw std::runtime_error("Data type handled in cast_compute_type_to_float");
|
||||
}
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/// Returns a status if cuDNN can satisfy a particular Conv2d description
|
||||
Status cudnn_satisfies(
|
||||
library::ConvDescription const &desc,
|
||||
library::Conv2dConfiguration const &configuration) {
|
||||
|
||||
auto const &a_tensor = desc.A;
|
||||
auto const &b_tensor = desc.B;
|
||||
auto const &c_tensor = desc.C;
|
||||
auto const &math_instruction = desc.tile_description.math_instruction;
|
||||
|
||||
if(a_tensor.element != b_tensor.element) {
|
||||
return Status::kErrorInvalidDataType;
|
||||
}
|
||||
|
||||
//////////////////////// Convolution output dimensions p and q ///////////////////////
|
||||
// Cutlass convolutions support arbitrary output dimensions and not constriant by //
|
||||
// input, filter, padding, striding, dilation sizes. //
|
||||
// cuDNN sets the output dimensions (p, q) using following equations: //
|
||||
// //
|
||||
// output = div_up(input + 2 * pad - ((filter - 1) * dilation + 1) + 1, stride) //
|
||||
// where; div_up(a, b) : (a - 1)/b + 1 //
|
||||
// //
|
||||
// Before launching cudnn verification or profiling check that output p and q //
|
||||
// dimensions are cuDNN compliant. //
|
||||
// //
|
||||
// If user sets output p and q which do not follow above constraints, cutlass conv, //
|
||||
// host reference, device reference can run. However, cudnn convolution returns //
|
||||
// "Invalid problem" //
|
||||
// //
|
||||
///////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// check conv output dimension p for cudnn
|
||||
int cudnn_output_p =
|
||||
(
|
||||
(
|
||||
configuration.problem_size.H +
|
||||
2 * configuration.problem_size.pad_h -
|
||||
((configuration.problem_size.R - 1) *
|
||||
configuration.problem_size.dilation_h + 1)
|
||||
) /
|
||||
(configuration.problem_size.stride_h)
|
||||
+ 1
|
||||
);
|
||||
|
||||
if (cudnn_output_p != configuration.problem_size.P) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
// check conv output dimension q for cudnn
|
||||
int cudnn_output_q =
|
||||
(
|
||||
(
|
||||
configuration.problem_size.W +
|
||||
2 * configuration.problem_size.pad_w -
|
||||
((configuration.problem_size.S - 1) *
|
||||
configuration.problem_size.dilation_w + 1)
|
||||
) /
|
||||
(configuration.problem_size.stride_w)
|
||||
+ 1
|
||||
);
|
||||
|
||||
if (cudnn_output_q != configuration.problem_size.Q) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// conv operator with input=FP16, accumulator=FP32, output=FP32 datatype
|
||||
if (a_tensor.element == library::NumericTypeID::kF16 &&
|
||||
b_tensor.element == library::NumericTypeID::kF16 &&
|
||||
math_instruction.element_accumulator == library::NumericTypeID::kF32 &&
|
||||
c_tensor.element == library::NumericTypeID::kF32
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
if (a_tensor.element == library::NumericTypeID::kBF16 ||
|
||||
b_tensor.element == library::NumericTypeID::kBF16 ||
|
||||
c_tensor.element == library::NumericTypeID::kBF16
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
// TF32 input not supported in cuDNN
|
||||
if (a_tensor.element == library::NumericTypeID::kTF32 ||
|
||||
b_tensor.element == library::NumericTypeID::kTF32 ||
|
||||
c_tensor.element == library::NumericTypeID::kTF32
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
if (a_tensor.element == library::NumericTypeID::kS8 ||
|
||||
b_tensor.element == library::NumericTypeID::kS8 ||
|
||||
c_tensor.element == library::NumericTypeID::kS8
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
if (a_tensor.element == library::NumericTypeID::kU8 ||
|
||||
b_tensor.element == library::NumericTypeID::kU8 ||
|
||||
c_tensor.element == library::NumericTypeID::kU8
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
if (a_tensor.element == library::NumericTypeID::kS4 ||
|
||||
b_tensor.element == library::NumericTypeID::kS4 ||
|
||||
c_tensor.element == library::NumericTypeID::kS4
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
if (a_tensor.element == library::NumericTypeID::kU4 ||
|
||||
b_tensor.element == library::NumericTypeID::kU4 ||
|
||||
c_tensor.element == library::NumericTypeID::kU4
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Returns a status if cuDNN can satisfy a particular Conv3d description
|
||||
Status cudnn_satisfies(
|
||||
library::ConvDescription const &desc,
|
||||
library::Conv3dConfiguration const &configuration) {
|
||||
|
||||
auto const &a_tensor = desc.A;
|
||||
auto const &b_tensor = desc.B;
|
||||
auto const &c_tensor = desc.C;
|
||||
auto const &math_instruction = desc.tile_description.math_instruction;
|
||||
|
||||
if(a_tensor.element != b_tensor.element) {
|
||||
return Status::kErrorInvalidDataType;
|
||||
}
|
||||
|
||||
//////////////////////// Convolution output dimensions p and q ///////////////////////
|
||||
// Cutlass convolutions support arbitrary output dimensions and not constriant by //
|
||||
// input, filter, padding, striding, dilation sizes. //
|
||||
// cuDNN sets the output dimensions (p, q) using following equations: //
|
||||
// //
|
||||
// output = div_up(input + 2 * pad - ((filter - 1) * dilation + 1) + 1, stride) //
|
||||
// where; div_up(a, b) : (a - 1)/b + 1 //
|
||||
// //
|
||||
// Before launching cudnn verification or profiling check that output p and q //
|
||||
// dimensions are cuDNN compliant. //
|
||||
// //
|
||||
// If user sets output p and q which do not follow above constraints, cutlass conv, //
|
||||
// host reference, device reference can run. However, cudnn convolution returns //
|
||||
// "Invalid problem" //
|
||||
// //
|
||||
///////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// check conv output dimension z for cudnn
|
||||
int cudnn_output_z =
|
||||
(
|
||||
(
|
||||
configuration.problem_size.D +
|
||||
2 * configuration.problem_size.pad_d -
|
||||
((configuration.problem_size.T - 1) *
|
||||
configuration.problem_size.dilation_d + 1)
|
||||
) /
|
||||
(configuration.problem_size.stride_d)
|
||||
+ 1
|
||||
);
|
||||
|
||||
if (cudnn_output_z != configuration.problem_size.Z) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
// check conv output dimension p for cudnn
|
||||
int cudnn_output_p =
|
||||
(
|
||||
(
|
||||
configuration.problem_size.H +
|
||||
2 * configuration.problem_size.pad_h -
|
||||
((configuration.problem_size.R - 1) *
|
||||
configuration.problem_size.dilation_h + 1)
|
||||
) /
|
||||
(configuration.problem_size.stride_h)
|
||||
+ 1
|
||||
);
|
||||
|
||||
if (cudnn_output_p != configuration.problem_size.P) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
// check conv output dimension q for cudnn
|
||||
int cudnn_output_q =
|
||||
(
|
||||
(
|
||||
configuration.problem_size.W +
|
||||
2 * configuration.problem_size.pad_w -
|
||||
((configuration.problem_size.S - 1) *
|
||||
configuration.problem_size.dilation_w + 1)
|
||||
) /
|
||||
(configuration.problem_size.stride_w)
|
||||
+ 1
|
||||
);
|
||||
|
||||
if (cudnn_output_q != configuration.problem_size.Q) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// conv operator with input, accumulator, output datatype of (hss) are not supported
|
||||
// in cuDNN
|
||||
if (a_tensor.element == library::NumericTypeID::kF16 &&
|
||||
b_tensor.element == library::NumericTypeID::kF16 &&
|
||||
math_instruction.element_accumulator == library::NumericTypeID::kF32 &&
|
||||
c_tensor.element == library::NumericTypeID::kF32
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
if (a_tensor.element == library::NumericTypeID::kBF16 ||
|
||||
b_tensor.element == library::NumericTypeID::kBF16 ||
|
||||
c_tensor.element == library::NumericTypeID::kBF16
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
if (a_tensor.element == library::NumericTypeID::kTF32 ||
|
||||
b_tensor.element == library::NumericTypeID::kTF32 ||
|
||||
c_tensor.element == library::NumericTypeID::kTF32
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
if (a_tensor.element == library::NumericTypeID::kS8 ||
|
||||
b_tensor.element == library::NumericTypeID::kS8 ||
|
||||
c_tensor.element == library::NumericTypeID::kS8
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
// S4 not supported in cuDNN
|
||||
if (a_tensor.element == library::NumericTypeID::kS4 ||
|
||||
b_tensor.element == library::NumericTypeID::kS4 ||
|
||||
c_tensor.element == library::NumericTypeID::kS4
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,584 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Helper functions for mapping CUTLASS concepts to cuDNN.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#if CUTLASS_ENABLE_CUDNN
|
||||
#include <cuda_runtime.h>
|
||||
#include <cudnn.h>
|
||||
#include <iostream>
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/util/device_memory.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "enumerated_types.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/// Converts a cuDNN status to cutlass::Status
|
||||
Status get_cutlass_status(cudnnStatus_t cudnn_status);
|
||||
|
||||
/// Converts a cuDNN status to cutlass::profiler::Disposition
|
||||
Disposition get_cutlass_disposition(cudnnStatus_t cudnn_status);
|
||||
|
||||
/// Checks cudnnStatus_t converts to cutlas status and returns if Status::kSuccess o.w. throws exception
|
||||
Status checkCudnnErr(cudnnStatus_t cudnn_status);
|
||||
|
||||
/// Maps a CUTLASS conv mode to a cuDNN conv mode enumeration
|
||||
bool get_cudnn_conv_mode(cudnnConvolutionMode_t &cudnn_conv_mode, conv::Mode conv_mode);
|
||||
|
||||
/// Maps a CUTLASS layout type to a cuDNN data type enumeration
|
||||
bool get_cudnn_layout(cudnnTensorFormat_t &cudnn_layout, library::LayoutTypeID layout);
|
||||
|
||||
/// Maps a CUTLASS numeric type to a cuDNN data type enumeration
|
||||
bool get_cudnn_datatype(cudnnDataType_t &cudnn_element_type, library::NumericTypeID element_type);
|
||||
|
||||
/// Maps CUTLASS math OpcodeClassID and MathOperationID to cuDNN math_type
|
||||
bool get_cudnn_mathtype(cudnnMathType_t &cudnn_math_type, library::ConvDescription const &conv_desc);
|
||||
|
||||
/// Returns a status if cudnn can satisfy a particular Conv2d description
|
||||
Status cudnn_satisfies(library::ConvDescription const &desc, library::Conv2dConfiguration const &configuration);
|
||||
|
||||
/// Returns a status if cudnn can satisfy a particular Conv3d description
|
||||
Status cudnn_satisfies(library::ConvDescription const &desc, library::Conv3dConfiguration const &configuration);
|
||||
|
||||
/// Cudnn compute type seems to be hardcoded to float (To handle a possible cudnn issue)
|
||||
float cast_cudnn_compute_type_to_float(library::NumericTypeID type, void const * src);
|
||||
|
||||
|
||||
/// This is a helper class to create cudnnHandle_t automatically on CudnnCreate object creation and
|
||||
/// to destroy cudnnHandle_t on CudnnCreate object destruction.
|
||||
/// Additionaly, it provides implicit cast from CudnnCreate's object to cudnnHandle_t's object
|
||||
class CudnnCreate {
|
||||
private:
|
||||
cudnnHandle_t handle;
|
||||
cudnnStatus_t status;
|
||||
|
||||
public:
|
||||
CudnnCreate() {
|
||||
status = cudnnCreate(&handle);
|
||||
}
|
||||
|
||||
~CudnnCreate() {
|
||||
cudnnDestroy(handle);
|
||||
}
|
||||
|
||||
/// Implicit cast CudnnCreate object to cudnnHandle_t
|
||||
operator cudnnHandle_t() const { return handle; }
|
||||
|
||||
/// returns cudnnStatus_t for handle creation
|
||||
cudnnStatus_t get_cudnn_create_status() { return status; }
|
||||
};
|
||||
|
||||
|
||||
namespace detail {
|
||||
|
||||
/// Dispatcher to cudnn convolution operators
|
||||
struct cudnnConvDispatcher {
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
//library::Conv2dConfiguration configuration;
|
||||
library::ConvArguments arguments;
|
||||
library::ConvKind conv_kind;
|
||||
|
||||
// cudnn-specific data structures to fill cudnn API call arguments
|
||||
// cudnn activation, filter, and output descriptors
|
||||
cudnnTensorDescriptor_t activation_desc;
|
||||
cudnnFilterDescriptor_t filter_desc;
|
||||
cudnnTensorDescriptor_t output_desc;
|
||||
cudnnConvolutionDescriptor_t conv_desc;
|
||||
|
||||
// cudnn datatypes
|
||||
cudnnDataType_t data_type_activation;
|
||||
cudnnDataType_t data_type_filter;
|
||||
cudnnDataType_t data_type_output;
|
||||
|
||||
// cudnn layouts
|
||||
cudnnTensorFormat_t layout_activation;
|
||||
cudnnTensorFormat_t layout_filter;
|
||||
cudnnTensorFormat_t layout_output;
|
||||
|
||||
// cudnn convolution mode
|
||||
cudnnConvolutionMode_t conv_mode;
|
||||
|
||||
// cudnn math type (tensorop, tensorop with conversion, simt)
|
||||
cudnnMathType_t math_type;
|
||||
|
||||
// cudnn compute data type
|
||||
cudnnDataType_t compute_type;
|
||||
|
||||
// cudnn compute type seems to be hardcoded to float (to handle a possible a cudnn issue)
|
||||
float alpha;
|
||||
float beta;
|
||||
|
||||
// cudnn workspace
|
||||
size_t workspace_size_in_bytes = 0;
|
||||
cutlass::device_memory::allocation<char> workspace;
|
||||
|
||||
// select cudnn's implicit gemm precomputed algorithm with tensor operations
|
||||
static cudnnConvolutionFwdAlgo_t const fprop_algo = CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM;
|
||||
static cudnnConvolutionBwdDataAlgo_t const dgrad_algo = CUDNN_CONVOLUTION_BWD_DATA_ALGO_1;
|
||||
static cudnnConvolutionBwdFilterAlgo_t const wgrad_algo = CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1;
|
||||
|
||||
Status status;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
// TODO: unify ctor cudnnConvDispatcher for conv2d and conv3d by unifying Conv2dConfigration
|
||||
|
||||
// ctor for conv2d
|
||||
cudnnConvDispatcher(
|
||||
library::ConvDescription const &op_desc,
|
||||
library::Conv2dConfiguration configuration,
|
||||
library::ConvArguments arguments_,
|
||||
cudnnHandle_t handle
|
||||
):
|
||||
//configuration(configuration_),
|
||||
arguments(arguments_),
|
||||
conv_kind(op_desc.conv_kind),
|
||||
status(Status::kSuccess) {
|
||||
|
||||
bool good = true;
|
||||
|
||||
// Get cudnn datatype, layout, and convolution mode from library::ConvDescription
|
||||
good = (good && get_cudnn_datatype(data_type_activation, op_desc.A.element));
|
||||
good = (good && get_cudnn_datatype(data_type_filter, op_desc.B.element));
|
||||
good = (good && get_cudnn_datatype(data_type_output, op_desc.C.element));
|
||||
good = (good && get_cudnn_layout(layout_activation, op_desc.A.layout));
|
||||
good = (good && get_cudnn_layout(layout_filter, op_desc.B.layout));
|
||||
good = (good && get_cudnn_layout(layout_output, op_desc.C.layout));
|
||||
good = (good && get_cudnn_conv_mode(conv_mode, configuration.problem_size.mode));
|
||||
// Get cudnn mathtype (cudnnMathType_t)
|
||||
good = (good && get_cudnn_mathtype(math_type, op_desc));
|
||||
good = (good && get_cudnn_datatype(
|
||||
compute_type,
|
||||
op_desc.tile_description.math_instruction.element_accumulator));
|
||||
// Check cutlass Conv2d description has equivalent operator in cudnn
|
||||
if (!good) {
|
||||
status = Status::kErrorNotSupported;
|
||||
return;
|
||||
}
|
||||
// cudnn compute type seems to be hardcoded to float (to handle a possible a cudnn issue)
|
||||
alpha = cast_cudnn_compute_type_to_float(op_desc.element_epilogue, arguments.alpha);
|
||||
beta = cast_cudnn_compute_type_to_float(op_desc.element_epilogue, arguments.beta);
|
||||
|
||||
// Create convolution descriptor object
|
||||
status = get_cutlass_status(cudnnCreateConvolutionDescriptor(&conv_desc));
|
||||
|
||||
// Configure convolution operator
|
||||
std::vector<int> padding {configuration.problem_size.pad_h, configuration.problem_size.pad_w};
|
||||
std::vector<int> stride {configuration.problem_size.stride_h, configuration.problem_size.stride_w};
|
||||
std::vector<int> dilation {configuration.problem_size.dilation_h, configuration.problem_size.dilation_w};
|
||||
|
||||
status = get_cutlass_status(
|
||||
cudnnSetConvolutionNdDescriptor(
|
||||
conv_desc,
|
||||
op_desc.conv_dim,
|
||||
padding.data(),
|
||||
stride.data(),
|
||||
dilation.data(),
|
||||
conv_mode,
|
||||
compute_type
|
||||
));
|
||||
|
||||
// Set groups
|
||||
status = get_cutlass_status(cudnnSetConvolutionGroupCount(conv_desc, configuration.problem_size.groups));
|
||||
|
||||
// Create activation, filter, and output descriptor objects
|
||||
status = get_cutlass_status(cudnnCreateTensorDescriptor(&activation_desc));
|
||||
status = get_cutlass_status(cudnnCreateFilterDescriptor(&filter_desc));
|
||||
status = get_cutlass_status(cudnnCreateTensorDescriptor(&output_desc));
|
||||
|
||||
// Set activation, filter, and output descriptor
|
||||
status = get_cutlass_status(
|
||||
cudnnSetTensor4dDescriptor(
|
||||
activation_desc,
|
||||
layout_activation,
|
||||
data_type_activation,
|
||||
configuration.problem_size.N,
|
||||
configuration.problem_size.C,
|
||||
configuration.problem_size.H,
|
||||
configuration.problem_size.W
|
||||
));
|
||||
|
||||
status = get_cutlass_status(
|
||||
cudnnSetFilter4dDescriptor(
|
||||
filter_desc,
|
||||
data_type_filter,
|
||||
layout_filter,
|
||||
configuration.problem_size.K,
|
||||
configuration.problem_size.C,
|
||||
configuration.problem_size.R,
|
||||
configuration.problem_size.S
|
||||
));
|
||||
|
||||
status = get_cutlass_status(
|
||||
cudnnSetTensor4dDescriptor(
|
||||
output_desc,
|
||||
layout_output,
|
||||
data_type_output,
|
||||
configuration.problem_size.N,
|
||||
configuration.problem_size.K,
|
||||
configuration.problem_size.P,
|
||||
configuration.problem_size.Q
|
||||
));
|
||||
|
||||
// Set math instruction to tensor op
|
||||
status = get_cutlass_status(
|
||||
cudnnSetConvolutionMathType(conv_desc, math_type));
|
||||
|
||||
// Initialize workspace
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop:
|
||||
status = get_cutlass_status(
|
||||
cudnnGetConvolutionForwardWorkspaceSize(
|
||||
handle,
|
||||
activation_desc,
|
||||
filter_desc,
|
||||
conv_desc,
|
||||
output_desc,
|
||||
fprop_algo,
|
||||
&workspace_size_in_bytes
|
||||
)); break;
|
||||
case library::ConvKind::kDgrad:
|
||||
status = get_cutlass_status(
|
||||
cudnnGetConvolutionBackwardDataWorkspaceSize(
|
||||
handle,
|
||||
filter_desc,
|
||||
output_desc,
|
||||
conv_desc,
|
||||
activation_desc,
|
||||
dgrad_algo,
|
||||
&workspace_size_in_bytes
|
||||
)); break;
|
||||
case library::ConvKind::kWgrad:
|
||||
status = get_cutlass_status(
|
||||
cudnnGetConvolutionBackwardFilterWorkspaceSize(
|
||||
handle,
|
||||
activation_desc,
|
||||
output_desc,
|
||||
conv_desc,
|
||||
filter_desc,
|
||||
wgrad_algo,
|
||||
&workspace_size_in_bytes
|
||||
)); break;
|
||||
|
||||
}
|
||||
|
||||
workspace = cutlass::device_memory::allocation<char>(workspace_size_in_bytes);
|
||||
}
|
||||
|
||||
|
||||
// ctor for conv3d
|
||||
cudnnConvDispatcher(
|
||||
library::ConvDescription const &op_desc,
|
||||
library::Conv3dConfiguration configuration,
|
||||
library::ConvArguments arguments_,
|
||||
cudnnHandle_t handle
|
||||
):
|
||||
//configuration(configuration_),
|
||||
arguments(arguments_),
|
||||
conv_kind(op_desc.conv_kind),
|
||||
status(Status::kSuccess) {
|
||||
|
||||
bool good = true;
|
||||
|
||||
// Get cudnn datatype, layout, and convolution mode from library::ConvDescription
|
||||
good = (good && get_cudnn_datatype(data_type_activation, op_desc.A.element));
|
||||
good = (good && get_cudnn_datatype(data_type_filter, op_desc.B.element));
|
||||
good = (good && get_cudnn_datatype(data_type_output, op_desc.C.element));
|
||||
|
||||
good = (good && get_cudnn_layout(layout_activation, op_desc.A.layout));
|
||||
good = (good && get_cudnn_layout(layout_filter, op_desc.B.layout));
|
||||
good = (good && get_cudnn_layout(layout_output, op_desc.C.layout));
|
||||
|
||||
good = (good && get_cudnn_conv_mode(conv_mode, configuration.problem_size.mode));
|
||||
|
||||
// cudnn compute type seems to be hardcoded to float (to handle a possible a cudnn issue)
|
||||
alpha = cast_cudnn_compute_type_to_float(op_desc.element_epilogue, arguments.alpha);
|
||||
beta = cast_cudnn_compute_type_to_float(op_desc.element_epilogue, arguments.beta);
|
||||
|
||||
good = (good && get_cudnn_datatype(
|
||||
compute_type,
|
||||
op_desc.tile_description.math_instruction.element_accumulator));
|
||||
|
||||
// Check cutlass Conv2d description has equivalent operator in cudnn
|
||||
if (!good) {
|
||||
status = Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
// Create convolution descriptor object
|
||||
status = get_cutlass_status(cudnnCreateConvolutionDescriptor(&conv_desc));
|
||||
|
||||
// Configure convolution operator
|
||||
std::vector<int> padding {configuration.problem_size.pad_d, configuration.problem_size.pad_h, configuration.problem_size.pad_w};
|
||||
std::vector<int> stride {configuration.problem_size.stride_d, configuration.problem_size.stride_h, configuration.problem_size.stride_w};
|
||||
std::vector<int> dilation {configuration.problem_size.dilation_d, configuration.problem_size.dilation_h, configuration.problem_size.dilation_w};
|
||||
|
||||
status = get_cutlass_status(
|
||||
cudnnSetConvolutionNdDescriptor(
|
||||
conv_desc,
|
||||
op_desc.conv_dim,
|
||||
padding.data(),
|
||||
stride.data(),
|
||||
dilation.data(),
|
||||
conv_mode,
|
||||
compute_type
|
||||
));
|
||||
|
||||
// Set groups
|
||||
status = get_cutlass_status(cudnnSetConvolutionGroupCount(conv_desc, configuration.problem_size.groups));
|
||||
|
||||
// Create activation, filter, and output descriptor objects
|
||||
status = get_cutlass_status(cudnnCreateTensorDescriptor(&activation_desc));
|
||||
status = get_cutlass_status(cudnnCreateFilterDescriptor(&filter_desc));
|
||||
status = get_cutlass_status(cudnnCreateTensorDescriptor(&output_desc));
|
||||
|
||||
// Set activation descriptor
|
||||
std::vector<int> activation_extent {
|
||||
configuration.problem_size.N,
|
||||
configuration.problem_size.C,
|
||||
configuration.problem_size.D,
|
||||
configuration.problem_size.H,
|
||||
configuration.problem_size.W
|
||||
};
|
||||
|
||||
std::vector<int> activation_stride {
|
||||
configuration.layout_activations.stride()[3],
|
||||
1,
|
||||
configuration.layout_activations.stride()[2],
|
||||
configuration.layout_activations.stride()[1],
|
||||
configuration.layout_activations.stride()[0]
|
||||
};
|
||||
|
||||
status = get_cutlass_status(
|
||||
cudnnSetTensorNdDescriptor(
|
||||
activation_desc,
|
||||
data_type_activation,
|
||||
op_desc.conv_dim + 2,
|
||||
activation_extent.data(),
|
||||
activation_stride.data()
|
||||
));
|
||||
|
||||
// Set filter descriptor
|
||||
std::vector<int> filter_extent {
|
||||
configuration.problem_size.K,
|
||||
configuration.problem_size.C,
|
||||
configuration.problem_size.T,
|
||||
configuration.problem_size.R,
|
||||
configuration.problem_size.S
|
||||
};
|
||||
|
||||
std::vector<int> filter_stride {
|
||||
configuration.layout_filters.stride()[3],
|
||||
1,
|
||||
configuration.layout_filters.stride()[2],
|
||||
configuration.layout_filters.stride()[1],
|
||||
configuration.layout_filters.stride()[0]
|
||||
};
|
||||
|
||||
status = get_cutlass_status(
|
||||
cudnnSetFilterNdDescriptor(
|
||||
filter_desc,
|
||||
data_type_filter,
|
||||
layout_filter,
|
||||
op_desc.conv_dim + 2,
|
||||
filter_extent.data()
|
||||
));
|
||||
|
||||
|
||||
// Set output descriptor
|
||||
std::vector<int> output_extent {
|
||||
configuration.problem_size.N,
|
||||
configuration.problem_size.K,
|
||||
configuration.problem_size.Z,
|
||||
configuration.problem_size.P,
|
||||
configuration.problem_size.Q
|
||||
};
|
||||
|
||||
std::vector<int> output_stride {
|
||||
configuration.layout_output.stride()[3],
|
||||
1,
|
||||
configuration.layout_output.stride()[2],
|
||||
configuration.layout_output.stride()[1],
|
||||
configuration.layout_output.stride()[0]
|
||||
};
|
||||
|
||||
status = get_cutlass_status(
|
||||
cudnnSetTensorNdDescriptor(
|
||||
output_desc,
|
||||
data_type_output,
|
||||
op_desc.conv_dim + 2,
|
||||
output_extent.data(),
|
||||
output_stride.data()
|
||||
));
|
||||
|
||||
// Set math instruction to tensor op
|
||||
status = get_cutlass_status(
|
||||
cudnnSetConvolutionMathType(conv_desc, math_type));
|
||||
|
||||
// Initialize workspace
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop:
|
||||
status = get_cutlass_status(
|
||||
cudnnGetConvolutionForwardWorkspaceSize(
|
||||
handle,
|
||||
activation_desc,
|
||||
filter_desc,
|
||||
conv_desc,
|
||||
output_desc,
|
||||
fprop_algo,
|
||||
&workspace_size_in_bytes
|
||||
)); break;
|
||||
case library::ConvKind::kDgrad:
|
||||
status = get_cutlass_status(
|
||||
cudnnGetConvolutionBackwardDataWorkspaceSize(
|
||||
handle,
|
||||
filter_desc,
|
||||
output_desc,
|
||||
conv_desc,
|
||||
activation_desc,
|
||||
dgrad_algo,
|
||||
&workspace_size_in_bytes
|
||||
)); break;
|
||||
case library::ConvKind::kWgrad:
|
||||
status = get_cutlass_status(
|
||||
cudnnGetConvolutionBackwardFilterWorkspaceSize(
|
||||
handle,
|
||||
activation_desc,
|
||||
output_desc,
|
||||
conv_desc,
|
||||
filter_desc,
|
||||
wgrad_algo,
|
||||
&workspace_size_in_bytes
|
||||
)); break;
|
||||
|
||||
}
|
||||
|
||||
workspace = cutlass::device_memory::allocation<char>(workspace_size_in_bytes);
|
||||
}
|
||||
|
||||
/// Executes Conv2d operater from cudnn library
|
||||
cudnnStatus_t operator()(cudnnHandle_t handle) {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop:
|
||||
return cudnnConvolutionForward(
|
||||
handle,
|
||||
&alpha,
|
||||
activation_desc,
|
||||
activation(),
|
||||
filter_desc,
|
||||
filter(),
|
||||
conv_desc,
|
||||
fprop_algo,
|
||||
workspace.get(),
|
||||
workspace_size_in_bytes,
|
||||
&beta,
|
||||
output_desc,
|
||||
arguments.D
|
||||
);
|
||||
case library::ConvKind::kDgrad:
|
||||
return cudnnConvolutionBackwardData(
|
||||
handle,
|
||||
&alpha,
|
||||
filter_desc,
|
||||
filter(),
|
||||
output_desc,
|
||||
output(),
|
||||
conv_desc,
|
||||
dgrad_algo,
|
||||
workspace.get(),
|
||||
workspace_size_in_bytes,
|
||||
&beta,
|
||||
activation_desc,
|
||||
arguments.D
|
||||
);
|
||||
case library::ConvKind::kWgrad:
|
||||
return cudnnConvolutionBackwardFilter(
|
||||
handle,
|
||||
&alpha,
|
||||
activation_desc,
|
||||
activation(),
|
||||
output_desc,
|
||||
output(),
|
||||
conv_desc,
|
||||
wgrad_algo,
|
||||
workspace.get(),
|
||||
workspace_size_in_bytes,
|
||||
&beta,
|
||||
filter_desc,
|
||||
arguments.D
|
||||
);
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns Actviation Tensor
|
||||
void const * activation() const {
|
||||
switch(conv_kind) {
|
||||
case library::ConvKind::kFprop : return arguments.A;
|
||||
case library::ConvKind::kDgrad : return arguments.C;
|
||||
case library::ConvKind::kWgrad : return arguments.B;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns Filter Tensor
|
||||
void const *filter() const {
|
||||
switch(conv_kind) {
|
||||
case library::ConvKind::kFprop : return arguments.B;
|
||||
case library::ConvKind::kDgrad : return arguments.B;
|
||||
case library::ConvKind::kWgrad : return arguments.C;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns Output Tensor
|
||||
void const *output() const {
|
||||
switch(conv_kind) {
|
||||
case library::ConvKind::kFprop : return arguments.C;
|
||||
case library::ConvKind::kDgrad : return arguments.A;
|
||||
case library::ConvKind::kWgrad : return arguments.A;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace detail
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#endif //#if CUTLASS_ENABLE_CUDNN
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
@@ -32,6 +32,8 @@
|
||||
// Profiler includes
|
||||
#include "cutlass_profiler.h"
|
||||
#include "gemm_operation_profiler.h"
|
||||
#include "conv2d_operation_profiler.h"
|
||||
#include "conv3d_operation_profiler.h"
|
||||
#include "sparse_gemm_operation_profiler.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -50,6 +52,10 @@ CutlassProfiler::CutlassProfiler(
|
||||
|
||||
operation_profilers_.emplace_back(new SparseGemmOperationProfiler(options));
|
||||
|
||||
operation_profilers_.emplace_back(new Conv2dOperationProfiler(options));
|
||||
|
||||
operation_profilers_.emplace_back(new Conv3dOperationProfiler(options));
|
||||
|
||||
}
|
||||
|
||||
CutlassProfiler::~CutlassProfiler() {
|
||||
@@ -159,6 +165,8 @@ void CutlassProfiler::print_usage_(std::ostream &out) {
|
||||
|
||||
out << "\n\nFor details about a particular function, specify the function name with --help.\n\nExample:\n\n"
|
||||
<< " $ cutlass_profiler --operation=Gemm --help\n\n"
|
||||
<< " $ cutlass_profiler --operation=Conv3d --help\n\n"
|
||||
<< " $ cutlass_profiler --operation=Conv2d --help\n\n"
|
||||
;
|
||||
}
|
||||
|
||||
|
||||
@@ -133,7 +133,18 @@ std::vector<int> DeviceAllocation::get_packed_layout(
|
||||
case library::LayoutTypeID::kTensorNDHWC:
|
||||
stride = get_packed_layout_stride<cutlass::layout::TensorNDHWC>(extent);
|
||||
break;
|
||||
|
||||
case library::LayoutTypeID::kTensorNC32HW32:
|
||||
stride = get_packed_layout_stride<cutlass::layout::TensorNCxHWx<32>>(extent);
|
||||
break;
|
||||
case library::LayoutTypeID::kTensorNC64HW64:
|
||||
stride = get_packed_layout_stride<cutlass::layout::TensorNCxHWx<64>>(extent);
|
||||
break;
|
||||
case library::LayoutTypeID::kTensorC32RSK32:
|
||||
stride = get_packed_layout_stride<cutlass::layout::TensorCxRSKx<32>>(extent);
|
||||
break;
|
||||
case library::LayoutTypeID::kTensorC64RSK64:
|
||||
stride = get_packed_layout_stride<cutlass::layout::TensorCxRSKx<64>>(extent);
|
||||
break;
|
||||
default: break;
|
||||
}
|
||||
|
||||
@@ -247,6 +258,18 @@ size_t DeviceAllocation::construct_layout(
|
||||
case library::LayoutTypeID::kTensorNDHWC:
|
||||
return construct_layout_<cutlass::layout::TensorNDHWC>(bytes, layout_id, extent, stride);
|
||||
|
||||
case library::LayoutTypeID::kTensorNC32HW32:
|
||||
return construct_layout_<cutlass::layout::TensorNCxHWx<32>>(bytes, layout_id, extent, stride);
|
||||
|
||||
case library::LayoutTypeID::kTensorNC64HW64:
|
||||
return construct_layout_<cutlass::layout::TensorNCxHWx<64>>(bytes, layout_id, extent, stride);
|
||||
|
||||
case library::LayoutTypeID::kTensorC32RSK32:
|
||||
return construct_layout_<cutlass::layout::TensorCxRSKx<32>>(bytes, layout_id, extent, stride);
|
||||
|
||||
case library::LayoutTypeID::kTensorC64RSK64:
|
||||
return construct_layout_<cutlass::layout::TensorCxRSKx<64>>(bytes, layout_id, extent, stride);
|
||||
|
||||
default: break;
|
||||
}
|
||||
|
||||
@@ -1362,6 +1385,18 @@ static void write_tensor_csv_static_type(
|
||||
case library::LayoutTypeID::kTensorNDHWC:
|
||||
write_tensor_csv_static_tensor_view<T, layout::TensorNDHWC>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kTensorNC32HW32:
|
||||
write_tensor_csv_static_tensor_view<T, layout::TensorNCxHWx<32>>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kTensorNC64HW64:
|
||||
write_tensor_csv_static_tensor_view<T, layout::TensorNCxHWx<64>>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kTensorC32RSK32:
|
||||
write_tensor_csv_static_tensor_view<T, layout::TensorCxRSKx<32>>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kTensorC64RSK64:
|
||||
write_tensor_csv_static_tensor_view<T, layout::TensorCxRSKx<64>>(out, allocation);
|
||||
break;
|
||||
default:
|
||||
throw std::runtime_error("Unhandled layout");
|
||||
}
|
||||
|
||||
@@ -243,7 +243,7 @@ int OperationProfiler::profile_all(
|
||||
ProblemSpace::Iterator problem_it = problem_space.begin();
|
||||
ProblemSpace::Iterator problem_end = problem_space.end();
|
||||
|
||||
bool continue_profiling = true;
|
||||
bool continue_profiling = true, internal_error = false;
|
||||
|
||||
// For each problem in problem space
|
||||
for (; continue_profiling && problem_it != problem_end; ++problem_it) {
|
||||
@@ -302,7 +302,8 @@ int OperationProfiler::profile_all(
|
||||
|
||||
if (status == Status::kErrorInternal) {
|
||||
// Stop profiling if there was an internal error
|
||||
return false;
|
||||
internal_error = true;
|
||||
break;
|
||||
}
|
||||
else if (status != Status::kSuccess) {
|
||||
// If the workspace could not be initialized for any other reason, continue to
|
||||
@@ -322,7 +323,8 @@ int OperationProfiler::profile_all(
|
||||
|
||||
if (status == Status::kErrorInternal) {
|
||||
// Stop profiling if there was an internal error
|
||||
return false;
|
||||
internal_error = true;
|
||||
break;
|
||||
}
|
||||
else if (status != Status::kSuccess) {
|
||||
// If the workspace could not be initialized for any other reason, continue to
|
||||
@@ -336,8 +338,9 @@ int OperationProfiler::profile_all(
|
||||
//
|
||||
|
||||
// B. Verify CUTLASS
|
||||
if (continue_profiling) {
|
||||
|
||||
|
||||
if (continue_profiling && options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
|
||||
|
||||
continue_profiling = this->verify_cutlass(
|
||||
options,
|
||||
report,
|
||||
@@ -368,6 +371,7 @@ int OperationProfiler::profile_all(
|
||||
//
|
||||
// D. Profile
|
||||
//
|
||||
|
||||
if (continue_profiling && options.profiling.enabled) {
|
||||
|
||||
continue_profiling = this->profile(
|
||||
@@ -392,10 +396,7 @@ int OperationProfiler::profile_all(
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Emit report
|
||||
report.close();
|
||||
|
||||
return 0;
|
||||
return internal_error ? 1 : 0;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -401,6 +401,7 @@ Options::Profiling::Profiling(cutlass::CommandLine const &cmdline) {
|
||||
else {
|
||||
providers.push_back(library::Provider::kCUTLASS);
|
||||
providers.push_back(library::Provider::kCUBLAS);
|
||||
providers.push_back(library::Provider::kCUDNN);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -428,8 +429,8 @@ void Options::Profiling::print_usage(std::ostream &out) const {
|
||||
|
||||
<< " --providers=<providers> "
|
||||
<< " List of providers to be profiled for performance. (default: '*')" << end_of_line
|
||||
<< " Gemm providers {cutlass*"
|
||||
<< "}" << end_of_line
|
||||
<< " Gemm providers {cutlass*, cublas*}" << end_of_line
|
||||
<< " Conv2d providers {cutlass*, cudnn*}"
|
||||
<< "\n\n";
|
||||
|
||||
}
|
||||
@@ -502,6 +503,7 @@ Options::Verification::Verification(cutlass::CommandLine const &cmdline) {
|
||||
else {
|
||||
providers.push_back(library::Provider::kCUBLAS);
|
||||
providers.push_back(library::Provider::kReferenceDevice);
|
||||
providers.push_back(library::Provider::kCUDNN);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -529,6 +531,7 @@ void Options::Verification::print_usage(std::ostream &out) const {
|
||||
<< " --verification-providers=<providers> "
|
||||
<< " List of providers used to verify result. (default: '*')" << end_of_line
|
||||
<< " Gemm verification-providers {cublas*}" << end_of_line
|
||||
<< " Conv2d verification-providers {cudnn*, device*, host}"
|
||||
<< "\n\n";
|
||||
}
|
||||
|
||||
@@ -570,6 +573,7 @@ Options::Report::Report(cutlass::CommandLine const &cmdline) {
|
||||
|
||||
cmdline.get_cmd_line_argument("append", append, false);
|
||||
cmdline.get_cmd_line_argument("output", output_path);
|
||||
cmdline.get_cmd_line_argument("junit-output", junit_output_path);
|
||||
|
||||
if (cmdline.check_cmd_line_flag("tags")) {
|
||||
cmdline.get_cmd_line_argument_pairs("tags", pivot_tags);
|
||||
@@ -591,6 +595,9 @@ void Options::Report::print_usage(std::ostream &out) const {
|
||||
<< " --output=<path> "
|
||||
<< " Path to output file for machine readable results. Operation kind and '.csv' is appended.\n\n"
|
||||
|
||||
<< " --junit-output=<path> "
|
||||
<< " Path to junit output file for result reporting. Operation kind and '.junit.xml' is appended.\n\n"
|
||||
|
||||
<< " --report-not-run=<bool> "
|
||||
<< " If true, reports the status of all kernels including those that" << end_of_line
|
||||
<< " do not satisfy the given arguments.\n\n"
|
||||
@@ -608,6 +615,7 @@ void Options::Report::print_options(std::ostream &out, int indent) const {
|
||||
out
|
||||
<< indent_str(indent) << "append: " << append << "\n"
|
||||
<< indent_str(indent) << "output: " << output_path << "\n"
|
||||
<< indent_str(indent) << "junit-output: " << junit_output_path << "\n"
|
||||
<< indent_str(indent) << "report_not_run: " << report_not_run << "\n"
|
||||
<< indent_str(indent) << "tags:\n";
|
||||
|
||||
|
||||
@@ -218,6 +218,9 @@ public:
|
||||
/// Path to a file containing results
|
||||
std::string output_path;
|
||||
|
||||
/// Path to a file containing junit xml results
|
||||
std::string junit_output_path;
|
||||
|
||||
/// Sequence of tags to attach to each result
|
||||
std::vector<std::pair<std::string, std::string>> pivot_tags;
|
||||
|
||||
|
||||
@@ -69,11 +69,15 @@ PerformanceReport::PerformanceReport(
|
||||
options_(options), argument_names_(argument_names), problem_index_(0), good_(true), op_kind_(op_kind) {
|
||||
|
||||
// Strip '.csv' if present
|
||||
std::string base_path = options_.report.output_path.substr(
|
||||
0, options_.report.output_path.rfind(".csv"));
|
||||
|
||||
std::string base_path = options_.report.output_path;
|
||||
base_path = base_path.substr(0, base_path.rfind(".csv"));
|
||||
op_file_name_ = base_path + "." + to_string(op_kind_) + ".csv";
|
||||
|
||||
base_path = options_.report.junit_output_path;
|
||||
base_path = base_path.substr(0, base_path.rfind(".xml"));
|
||||
base_path = base_path.substr(0, base_path.rfind(".junit"));
|
||||
op_junit_file_name_ = base_path + "." + to_string(op_kind_) + ".junit.xml";
|
||||
|
||||
//
|
||||
// Open output file for operation of PerformanceReport::op_kind
|
||||
//
|
||||
@@ -108,6 +112,21 @@ PerformanceReport::PerformanceReport(
|
||||
print_csv_header_(output_file_) << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
if (!options_.report.junit_output_path.empty()) {
|
||||
|
||||
junit_output_file_.open(op_junit_file_name_);
|
||||
|
||||
if (!junit_output_file_.good()) {
|
||||
|
||||
std::cerr << "Could not open junit output file at path '"
|
||||
<< options_.report.junit_output_path << "'" << std::endl;
|
||||
|
||||
good_ = false;
|
||||
}
|
||||
|
||||
print_junit_header_(junit_output_file_);
|
||||
}
|
||||
}
|
||||
|
||||
void PerformanceReport::next_problem() {
|
||||
@@ -123,6 +142,10 @@ void PerformanceReport::append_result(PerformanceResult result) {
|
||||
print_result_pretty_(std::cout, result) << std::flush;
|
||||
}
|
||||
|
||||
if (junit_output_file_.is_open()) {
|
||||
print_junit_result_(junit_output_file_, result);
|
||||
}
|
||||
|
||||
if (output_file_.is_open()) {
|
||||
print_result_csv_(output_file_, result) << std::endl;
|
||||
}
|
||||
@@ -143,7 +166,7 @@ void PerformanceReport::append_results(PerformanceResultVector const &results) {
|
||||
}
|
||||
}
|
||||
|
||||
void PerformanceReport::close() {
|
||||
PerformanceReport::~PerformanceReport() {
|
||||
|
||||
//
|
||||
// Output results to stdout if they were not written to a file already.
|
||||
@@ -161,7 +184,17 @@ void PerformanceReport::close() {
|
||||
}
|
||||
}
|
||||
else if (output_file_.is_open() && options_.report.verbose) {
|
||||
std::cout << "\n\nWrote results to '" << op_file_name_ << "'" << std::endl;
|
||||
std::cout << "\nWrote results to '" << op_file_name_ << "'" << std::endl;
|
||||
}
|
||||
|
||||
if (output_file_.is_open()) {
|
||||
output_file_.close();
|
||||
}
|
||||
|
||||
if (junit_output_file_.is_open()) {
|
||||
print_junit_footer_(junit_output_file_);
|
||||
junit_output_file_.close();
|
||||
std::cout << "\nWrote jUnit results to '" << op_junit_file_name_ << "'" << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -179,7 +212,8 @@ static const char *disposition_status_color(Disposition disposition) {
|
||||
/// Prints the result in human readable form
|
||||
std::ostream & PerformanceReport::print_result_pretty_(
|
||||
std::ostream &out,
|
||||
PerformanceResult const &result) {
|
||||
PerformanceResult const &result,
|
||||
bool use_shell_coloring) {
|
||||
|
||||
out << "=============================\n"
|
||||
<< " Problem ID: " << result.problem_index << "\n";
|
||||
@@ -196,14 +230,20 @@ std::ostream & PerformanceReport::print_result_pretty_(
|
||||
out << "\n";
|
||||
}
|
||||
|
||||
std::string shell_color_bright = use_shell_coloring ? SHELL_COLOR_BRIGHT() : "";
|
||||
std::string shell_color_end = use_shell_coloring ? SHELL_COLOR_END() : "";
|
||||
auto _disposition_status_color = [&](Disposition d) -> const char * {
|
||||
return use_shell_coloring ? disposition_status_color(d) : "";
|
||||
};
|
||||
|
||||
out
|
||||
<< "\n"
|
||||
<< " Provider: " << SHELL_COLOR_BRIGHT() << library::to_string(result.provider, true) << SHELL_COLOR_END() << "\n"
|
||||
<< " OperationKind: " << SHELL_COLOR_BRIGHT() << library::to_string(result.op_kind) << SHELL_COLOR_END() << "\n"
|
||||
<< " Provider: " << shell_color_bright << library::to_string(result.provider, true) << shell_color_end << "\n"
|
||||
<< " OperationKind: " << shell_color_bright << library::to_string(result.op_kind) << shell_color_end << "\n"
|
||||
<< " Operation: " << result.operation_name << "\n\n"
|
||||
<< " Status: " << SHELL_COLOR_BRIGHT() << library::to_string(result.status, true) << SHELL_COLOR_END() << "\n"
|
||||
<< " Verification: " << SHELL_COLOR_BRIGHT() << (options_.verification.enabled ? "ON":"OFF") << SHELL_COLOR_END() << "\n"
|
||||
<< " Disposition: " << disposition_status_color(result.disposition) << to_string(result.disposition, true) << SHELL_COLOR_END() << "\n\n";
|
||||
<< " Status: " << shell_color_bright << library::to_string(result.status, true) << shell_color_end << "\n"
|
||||
<< " Verification: " << shell_color_bright << (options_.verification.enabled ? "ON":"OFF") << shell_color_end << "\n"
|
||||
<< " Disposition: " << _disposition_status_color(result.disposition) << to_string(result.disposition, true) << shell_color_end << "\n\n";
|
||||
|
||||
// Display individual verification results for each verification-provider
|
||||
if (options_.verification.enabled) {
|
||||
@@ -263,10 +303,6 @@ std::ostream & PerformanceReport::print_csv_header_(
|
||||
<< ",OperationKind,Operation,Disposition,Status";
|
||||
|
||||
for (auto const &arg_name : argument_names_) {
|
||||
// Operand E is internal to the sparse kernel
|
||||
if (arg_name.compare("E") == 0)
|
||||
continue;
|
||||
|
||||
out << "," << arg_name;
|
||||
}
|
||||
|
||||
@@ -327,6 +363,112 @@ std::ostream & PerformanceReport::print_result_csv_(
|
||||
return out;
|
||||
}
|
||||
|
||||
std::ostream & PerformanceReport::print_junit_header_(std::ostream &out) {
|
||||
|
||||
out << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" << std::endl;
|
||||
out << "<testsuite name=\"cutlass_profiler\">" << std::endl;
|
||||
return out;
|
||||
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
std::string escape_xml_special_chars(const std::string& src) {
|
||||
std::stringstream dst;
|
||||
for (char ch : src) {
|
||||
switch (ch) {
|
||||
case '&': dst << "&"; break;
|
||||
case '\'': dst << "'"; break;
|
||||
case '"': dst << """; break;
|
||||
case '<': dst << "<"; break;
|
||||
case '>': dst << ">"; break;
|
||||
default: dst << ch; break;
|
||||
}
|
||||
}
|
||||
return dst.str();
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
std::ostream & print_junit_result_property_(std::ostream & os, const std::string & name, const T & property) {
|
||||
return os << " <property name=\"" << name << "\" value=\"" << property << "\" />" << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
std::ostream & PerformanceReport::print_junit_result_(std::ostream &out, PerformanceResult const &result) {
|
||||
|
||||
out << " " << "<testcase name=\"";
|
||||
|
||||
std::string delim = "";
|
||||
|
||||
// Pivot tags
|
||||
for (auto const & tag : options_.report.pivot_tags) {
|
||||
out << delim << tag.second; delim = "_";
|
||||
}
|
||||
|
||||
out << delim << to_string(result.op_kind); delim = "_";
|
||||
out << delim << result.operation_name;
|
||||
|
||||
for (auto const & arg : result.arguments) {
|
||||
out << delim << arg.second;
|
||||
}
|
||||
|
||||
out << "\" ";
|
||||
|
||||
bool skipped = false, failed = false, error = false;
|
||||
|
||||
switch (result.disposition) {
|
||||
case Disposition::kNotRun:
|
||||
case Disposition::kNotSupported:
|
||||
skipped = true;
|
||||
break;
|
||||
case Disposition::kPassed:
|
||||
case Disposition::kNotVerified:
|
||||
break;
|
||||
case Disposition::kFailed:
|
||||
case Disposition::kIncorrect:
|
||||
failed = true;
|
||||
break;
|
||||
case Disposition::kInvalidProblem:
|
||||
case Disposition::kInvalid:
|
||||
error = true;
|
||||
break;
|
||||
};
|
||||
|
||||
if (skipped) {
|
||||
out << "status=\"notrun\"";
|
||||
} else {
|
||||
out << "status=\"run\"";
|
||||
}
|
||||
|
||||
out << ">" << std::endl;
|
||||
|
||||
if (failed) {
|
||||
out << " <failure message=\"" << to_string(result.disposition) << "\" />" << std::endl;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
out << " <error message=\"" << to_string(result.disposition) << "\" />" << std::endl;
|
||||
}
|
||||
|
||||
out << " <system-out><![CDATA[" << std::endl;
|
||||
std::stringstream ss;
|
||||
print_result_pretty_(ss, result, false);
|
||||
out << escape_xml_special_chars(ss.str()) << std::endl;
|
||||
out << " ]]></system-out>" << std::endl;
|
||||
|
||||
out << " </testcase>" << std::endl;
|
||||
|
||||
return out;
|
||||
|
||||
}
|
||||
|
||||
std::ostream & PerformanceReport::print_junit_footer_(std::ostream &out) {
|
||||
|
||||
out << "</testsuite>" << std::endl;
|
||||
return out;
|
||||
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
|
||||
@@ -59,6 +59,12 @@ private:
|
||||
/// Output file containing results
|
||||
std::ofstream output_file_;
|
||||
|
||||
/// Operation file name containing junit performance report of op_kind
|
||||
std::string op_junit_file_name_;
|
||||
|
||||
/// Output file containing junit results
|
||||
std::ofstream junit_output_file_;
|
||||
|
||||
/// Flag indicating the performance report is valid
|
||||
bool good_;
|
||||
|
||||
@@ -74,6 +80,7 @@ private:
|
||||
public:
|
||||
|
||||
PerformanceReport(Options const &options, std::vector<std::string> const &argument_names, library::OperationKind const &op_kind);
|
||||
~PerformanceReport();
|
||||
|
||||
bool good() const { return good_; }
|
||||
|
||||
@@ -81,8 +88,6 @@ public:
|
||||
void append_result(PerformanceResult result);
|
||||
void append_results(PerformanceResultVector const &results);
|
||||
|
||||
void close();
|
||||
|
||||
public:
|
||||
|
||||
/// Prints the CSV header
|
||||
@@ -91,10 +96,21 @@ public:
|
||||
/// Prints the CSV
|
||||
std::ostream & print_result_csv_(std::ostream &out, PerformanceResult const &result);
|
||||
|
||||
/// @defgroup jUnit Result Generation
|
||||
/// Functions related to generation of the jUnit results
|
||||
/// @{
|
||||
|
||||
std::ostream & print_junit_header_(std::ostream &out);
|
||||
std::ostream & print_junit_result_(std::ostream &out, PerformanceResult const &result);
|
||||
std::ostream & print_junit_footer_(std::ostream &out);
|
||||
|
||||
/// @}
|
||||
|
||||
/// Prints the result in human readable form
|
||||
std::ostream & print_result_pretty_(
|
||||
std::ostream &out,
|
||||
PerformanceResult const &result);
|
||||
PerformanceResult const &result,
|
||||
bool use_shell_coloring = true);
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -961,6 +961,85 @@ bool arg_as_SplitKModeID(
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_ConvModeID(
|
||||
library::ConvModeID &conv_mode,
|
||||
KernelArgument::Value const *value_ptr) {
|
||||
|
||||
if (value_ptr->not_null) {
|
||||
if (value_ptr->argument->description->type == ArgumentTypeID::kEnumerated) {
|
||||
|
||||
conv_mode = library::from_string<library::ConvModeID>(
|
||||
static_cast<EnumeratedTypeArgument::EnumeratedTypeValue const *>(value_ptr)->element);
|
||||
|
||||
if (conv_mode == library::ConvModeID::kInvalid) {
|
||||
throw std::runtime_error(
|
||||
"arg_as_ConvModeID() - illegal cast.");
|
||||
}
|
||||
}
|
||||
else {
|
||||
|
||||
throw std::runtime_error(
|
||||
"arg_as_ConvModeID() - illegal cast.");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_ConvModeID(
|
||||
library::ConvModeID &conv_mode,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
size_t idx = problem_space.argument_index(name);
|
||||
KernelArgument::Value const *value_ptr = problem.at(idx).get();
|
||||
|
||||
return arg_as_ConvModeID(conv_mode, value_ptr);
|
||||
}
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_ProviderID(
|
||||
library::Provider &provider,
|
||||
KernelArgument::Value const *value_ptr) {
|
||||
|
||||
if (value_ptr->not_null) {
|
||||
if (value_ptr->argument->description->type == ArgumentTypeID::kEnumerated) {
|
||||
|
||||
provider = library::from_string<library::Provider>(
|
||||
static_cast<EnumeratedTypeArgument::EnumeratedTypeValue const *>(value_ptr)->element);
|
||||
|
||||
if (provider == library::Provider::kInvalid) {
|
||||
throw std::runtime_error(
|
||||
"arg_as_ProviderID() - illegal cast.");
|
||||
}
|
||||
}
|
||||
else {
|
||||
|
||||
throw std::runtime_error(
|
||||
"arg_as_ProviderID() - illegal cast.");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_ProviderID(
|
||||
library::Provider &provider,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
size_t idx = problem_space.argument_index(name);
|
||||
KernelArgument::Value const *value_ptr = problem.at(idx).get();
|
||||
|
||||
return arg_as_ProviderID(provider, value_ptr);
|
||||
}
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Lexically casts an argument to a given type stored in a byte array. Returns true if not null.
|
||||
bool arg_as_scalar(
|
||||
std::vector<uint8_t> &bytes,
|
||||
@@ -1049,9 +1128,94 @@ bool tensor_description_satisfies(
|
||||
return false;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Returns true if conv_kind satisfies the value
|
||||
bool conv_kind_satisfies(
|
||||
library::ConvKind const &conv_kind,
|
||||
EnumeratedTypeArgument::EnumeratedTypeValue const *value_ptr) {
|
||||
|
||||
if (value_ptr->not_null) {
|
||||
library::ConvKind conv_kind_cmd_line =
|
||||
library::from_string<library::ConvKind>(value_ptr->element);
|
||||
|
||||
if (conv_kind_cmd_line != library::ConvKind::kUnknown &&
|
||||
conv_kind_cmd_line != conv_kind) {
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Returns true if conv_kind satisfies the value
|
||||
bool conv_kind_satisfies(
|
||||
library::ConvKind const &conv_kind,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
size_t idx = problem_space.argument_index(name);
|
||||
KernelArgument::Value const *value_ptr = problem.at(idx).get();
|
||||
|
||||
if (value_ptr->argument->description->type == ArgumentTypeID::kEnumerated) {
|
||||
return conv_kind_satisfies(
|
||||
conv_kind,
|
||||
static_cast<EnumeratedTypeArgument::EnumeratedTypeValue const *>(value_ptr));
|
||||
}
|
||||
else {
|
||||
throw std::runtime_error("Kernel argument mismatch");
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Returns true if a iterator algorithm satisfies the value
|
||||
bool iterator_algorithm_satisfies(
|
||||
library::IteratorAlgorithmID const &iterator_algorithm,
|
||||
EnumeratedTypeArgument::EnumeratedTypeValue const *value_ptr) {
|
||||
|
||||
if (value_ptr->not_null) {
|
||||
library::IteratorAlgorithmID iterator_algorithm_cmd_line =
|
||||
library::from_string<library::IteratorAlgorithmID>(value_ptr->element);
|
||||
|
||||
if (iterator_algorithm_cmd_line != library::IteratorAlgorithmID::kNone &&
|
||||
iterator_algorithm_cmd_line != iterator_algorithm) {
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Returns true if a iterator algorithm satisfies the value
|
||||
bool iterator_algorithm_satisfies(
|
||||
library::IteratorAlgorithmID const &iterator_algorithm,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
size_t idx = problem_space.argument_index(name);
|
||||
KernelArgument::Value const *value_ptr = problem.at(idx).get();
|
||||
|
||||
if (value_ptr->argument->description->type == ArgumentTypeID::kEnumerated) {
|
||||
return iterator_algorithm_satisfies(
|
||||
iterator_algorithm,
|
||||
static_cast<EnumeratedTypeArgument::EnumeratedTypeValue const *>(value_ptr));
|
||||
}
|
||||
else {
|
||||
throw std::runtime_error("Kernel argument mismatch");
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -909,6 +909,37 @@ bool arg_as_SplitKModeID(
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_ConvModeID(library::ConvModeID &conv_mode, KernelArgument::Value const *value_ptr);
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_ConvModeID(
|
||||
library::ConvModeID &conv_mode,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_IteratorAlgorithmID(library::IteratorAlgorithmID &iterator_algorithm, KernelArgument::Value const *value_ptr);
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_IteratorAlgorithmID(
|
||||
library::IteratorAlgorithmID &iterator_algorithm,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_ProviderID(library::Provider &provider, KernelArgument::Value const *value_ptr);
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_ProviderID(
|
||||
library::Provider &provider,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Lexically casts an argument to a given type stored in a byte array. Returns true if not null.
|
||||
bool arg_as_scalar(
|
||||
std::vector<uint8_t> &bytes,
|
||||
@@ -935,10 +966,34 @@ bool tensor_description_satisfies(
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
|
||||
/// Returns true if a conv kind satisfies the value
|
||||
bool conv_kind_satisfies(
|
||||
library::ConvKind const &conv_kind,
|
||||
EnumeratedTypeArgument::EnumeratedTypeValue const *value_ptr);
|
||||
|
||||
/// Returns true if a conv kind satisfies the value
|
||||
bool conv_kind_satisfies(
|
||||
library::ConvKind const &conv_kind,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Returns true if a iterator algorithm satisfies the value
|
||||
bool iterator_algorithm_satisfies(
|
||||
library::IteratorAlgorithmID const &iterator_algorithm,
|
||||
EnumeratedTypeArgument::EnumeratedTypeValue const *value_ptr);
|
||||
|
||||
/// Returns true if a iterator algorithm satisfies the value
|
||||
bool iterator_algorithm_satisfies(
|
||||
library::IteratorAlgorithmID const &iterator_algorithm,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Defines profiling functionality for reduction operation
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <algorithm>
|
||||
#include <unordered_map>
|
||||
|
||||
// CUTLASS Library includes
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/util.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
// Profiler includes
|
||||
#include "options.h"
|
||||
#include "device_context.h"
|
||||
#include "operation_profiler.h"
|
||||
#include "performance_result.h"
|
||||
#include "problem_space.h"
|
||||
#if CUTLASS_ENABLE_CUDNN
|
||||
#include "cudnn_helpers.h"
|
||||
#endif //#if CUTLASS_ENABLE_CUDNN
|
||||
#include "debug.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Abstract base class for each math function
|
||||
class ReductionOperationProfiler : public OperationProfiler {
|
||||
public:
|
||||
|
||||
|
||||
/// Workspace used
|
||||
struct ReductionWorkspace {
|
||||
|
||||
/// Conv device allocations
|
||||
DeviceAllocation *Workspace;
|
||||
DeviceAllocation *Source;
|
||||
DeviceAllocation *Destination;
|
||||
DeviceAllocation *Reference;
|
||||
|
||||
/// Library configuration and arguments
|
||||
library::ReductionConfiguration configuration;
|
||||
library::ReductionArguments arguments;
|
||||
|
||||
/// Buffer used for the cutlass operations' host workspace
|
||||
std::vector<uint8_t> host_workspace;
|
||||
|
||||
/// Buffer used for the cutlass operations' device workspace
|
||||
DeviceAllocation device_workspace;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
ReductionWorkspace():
|
||||
Workspace(nullptr), Source(nullptr), Destination(nullptr), Reference(nullptr) { }
|
||||
};
|
||||
|
||||
protected:
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// Reduction problem obtained from problem space
|
||||
MatrixCoord problem_;
|
||||
|
||||
/// Device memory allocations
|
||||
ReductionWorkspace conv_workspace_;
|
||||
|
||||
|
||||
public:
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Ctor
|
||||
ReductionOperationProfiler(Options const &options);
|
||||
|
||||
/// Destructor
|
||||
virtual ~ReductionOperationProfiler();
|
||||
|
||||
/// Prints usage statement for the math function
|
||||
virtual void print_usage(std::ostream &out) const;
|
||||
|
||||
/// Prints examples
|
||||
virtual void print_examples(std::ostream &out) const;
|
||||
|
||||
/// Extracts the problem dimensions
|
||||
virtual Status initialize_configuration(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Initializes workspace
|
||||
virtual Status initialize_workspace(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
virtual bool verify_cutlass(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Measures performance results
|
||||
virtual bool profile(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -227,6 +227,9 @@ void SparseGemmOperationProfiler::SparseGemmProblem::initialize_result(
|
||||
set_argument(result, "C", problem_space,
|
||||
std::string(library::to_string(operation_desc.C.element)) + ":" + library::to_string(operation_desc.C.layout));
|
||||
|
||||
set_argument(result, "E", problem_space,
|
||||
std::string(library::to_string(operation_desc.E.element)) + ":" + library::to_string(operation_desc.E.layout));
|
||||
|
||||
set_argument(result, "m", problem_space, m);
|
||||
set_argument(result, "n", problem_space, n);
|
||||
set_argument(result, "k", problem_space, k);
|
||||
|
||||
Reference in New Issue
Block a user