leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

CUTLASS 3.2.1 (#1113)

Browse Source 
* Updates for 3.2.1 release.

* Minor fix in gemm op profiler for raster order.

* Add scheduler mapping for raster order in the kernels.
This commit is contained in:
ANIKET SHIVAM
2023-09-26 14:24:26 -07:00
committed by GitHub
parent e0aaa3c3b3
commit 90d3b0fb18
428 changed files with 22253 additions and 21762 deletions
Download Patch File Download Diff File Expand all files Collapse all files

495
tools/profiler/include/cutlass/profiler/conv2d_operation_profiler.h Normal file
View File

@@ -0,0 +1,495 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Defines profiling functionality for convolution
*/
#pragma once
#include <vector>
#include <string>
#include <memory>
#include <algorithm>
#include <unordered_map>
// CUTLASS Library includes
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
#include "cutlass/library/handle.h"
#include "cutlass/library/manifest.h"
#include "cutlass/library/singleton.h"
// Profiler includes
#include "options.h"
#include "device_context.h"
#include "operation_profiler.h"
#include "performance_result.h"
#include "problem_space.h"
#include "reduction_operation_profiler.h"
#if CUTLASS_ENABLE_CUDNN
#include "cudnn_helpers.h"
#endif //#if CUTLASS_ENABLE_CUDNN
#include "debug.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Abstract base class for each math function
class Conv2dOperationProfiler : public OperationProfiler {
public:
/// Problem structure obtained from problem space
struct Conv2dProblem {
int64_t n, h, w, c, p, q, k, r, s;
int64_t groups;
int64_t pad_h, pad_w;
int64_t stride_h, stride_w;
int64_t dilation_h, dilation_w;
std::vector<uint8_t> alpha;
std::vector<uint8_t> beta;
library::SplitKMode split_k_mode;
int64_t split_k_slices;
library::ConvModeID conv_mode;
library::Provider eq_gemm_provider;
// convolution with parallel interleaved reduction
// convolution epilogue (alpha, beta) = (1.0, 0.0)
// reduction epilogue (alpha, beta) = (Conv2dProblem::alpha, Conv2dProblem::beta)
std::vector<uint8_t> alpha_one;
std::vector<uint8_t> beta_zero;
//
// Methods
//
/// Total number of bytes loaded
int64_t bytes(library::ConvDescription const &operation_desc) const;
/// Total number of flops computed
int64_t flops(library::ConvDescription const &operation_desc) const;
void set_default_output_size() {
p = ((h + pad_h - r * dilation_h) / stride_h) + 1;
q = ((w + pad_w - s * dilation_w) / stride_w) + 1;
}
// Returns equivalent gemm problem size for convolution
cutlass::gemm::GemmCoord eq_gemm_size(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return cutlass::gemm::GemmCoord(int(n * p * q), int(k), int(r * s * c / groups));
case library::ConvKind::kDgrad: return cutlass::gemm::GemmCoord(int(n * h * w), int(c), int(k * r * s));
case library::ConvKind::kWgrad: return cutlass::gemm::GemmCoord(int(k), int(r * s * c), int(n * p * q));
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns extent for tensor A
std::vector<int> extent_a(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return {int(n), int(h), int(w), int(c)};
case library::ConvKind::kDgrad: return {int(n), int(p), int(q), int(k)};
case library::ConvKind::kWgrad: return {int(n), int(p), int(q), int(k)};
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns extent for tensor B
std::vector<int> extent_b(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return {int(k), int(r), int(s), int(c / groups)};
case library::ConvKind::kDgrad: return {int(k), int(r), int(s), int(c)};
case library::ConvKind::kWgrad: return {int(n), int(h), int(w), int(c)};
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns extent for tensor C
std::vector<int> extent_c(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return {int(n), int(p), int(q), int(k)};
case library::ConvKind::kDgrad: return {int(n), int(h), int(w), int(c)};
case library::ConvKind::kWgrad: return {int(k), int(r), int(s), int(c)};
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns layout for equivalent gemm matrix A
library::LayoutTypeID eq_gemm_layout_a(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return library::LayoutTypeID::kRowMajor; // TN Gemm
case library::ConvKind::kDgrad: return library::LayoutTypeID::kRowMajor; // TT Gemm
case library::ConvKind::kWgrad: return library::LayoutTypeID::kColumnMajor; // NT Gemm
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns layout for equivalent gemm matrix B
library::LayoutTypeID eq_gemm_layout_b(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return library::LayoutTypeID::kColumnMajor; // TN Gemm
case library::ConvKind::kDgrad: return library::LayoutTypeID::kRowMajor; // TT Gemm
case library::ConvKind::kWgrad: return library::LayoutTypeID::kRowMajor; // NT Gemm
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns layout for equivalent gemm matrix C
library::LayoutTypeID eq_gemm_layout_c(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
// Gemm operator assumes column-major output
case library::ConvKind::kFprop:
case library::ConvKind::kDgrad:
case library::ConvKind::kWgrad: return library::LayoutTypeID::kColumnMajor;
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns leading dimension for equivalent gemm matrix A
int64_t eq_gemm_lda(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return eq_gemm_size(conv_kind).k();
case library::ConvKind::kDgrad: return eq_gemm_size(conv_kind).k();
case library::ConvKind::kWgrad: return eq_gemm_size(conv_kind).m();
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns leading dimension for equivalent gemm matrix B
int64_t eq_gemm_ldb(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return eq_gemm_size(conv_kind).k();
case library::ConvKind::kDgrad: return eq_gemm_size(conv_kind).n();
case library::ConvKind::kWgrad: return eq_gemm_size(conv_kind).n();
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns leading dimension for equivalent gemm matrix C
int64_t eq_gemm_ldc(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop:
case library::ConvKind::kDgrad:
case library::ConvKind::kWgrad: return eq_gemm_size(conv_kind).m();
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
};
/// Workspace used
struct Conv2dWorkspace {
/// Conv device allocations
DeviceAllocation *A;
DeviceAllocation *B;
DeviceAllocation *reordered_B;
DeviceAllocation *C;
DeviceAllocation *Computed;
DeviceAllocation *Reference;
/// Library configuration and arguments for convolution operator
library::Conv2dConfiguration configuration;
library::ConvArguments arguments;
/// Number of copies of the problem workspace which are visited sequentially during
/// profiling to avoid camping in the last level cache.
int problem_count;
/// Buffer used for the cutlass conv2d operations' host workspace
std::vector<uint8_t> host_workspace;
/// Buffer used for the cutlass operations' device workspace
DeviceAllocation device_workspace;
/// Library configuration and arguments for reduction operator
library::ReductionConfiguration reduction_configuration;
library::ReductionArguments reduction_arguments;
/// Buffer used for the cutlass reduction operations' host workspace
std::vector<uint8_t> reduction_host_workspace;
/// Host data buffers for host reference operation
/// host buffer for tensor
std::vector<uint8_t> host_tensor_a;
/// host buffer for tensor b
std::vector<uint8_t> host_tensor_b;
/// host buffer for tensor c
std::vector<uint8_t> host_tensor_c;
//
// Methods
//
Conv2dWorkspace()
: A(nullptr),
B(nullptr),
reordered_B(nullptr),
C(nullptr),
Computed(nullptr),
Reference(nullptr) {}
// Set stride vector for tensor activations, filters, output
void set_stride_vector(Conv2dProblem const &problem,
library::ConvKind const &conv_kind,
library::LayoutTypeID const &layout_a,
library::LayoutTypeID const &layout_b,
library::LayoutTypeID const &layout_c) {
std::vector<int64_t> stride_activations;
std::vector<int64_t> stride_filters;
std::vector<int64_t> stride_output;
// Strides for interleaved fprop
if (conv_kind == library::ConvKind::kFprop &&
((layout_a == library::LayoutTypeID::kTensorNC32HW32 &&
layout_b == library::LayoutTypeID::kTensorC32RSK32 &&
layout_c == library::LayoutTypeID::kTensorNC32HW32) ||
(layout_a == library::LayoutTypeID::kTensorNC64HW64 &&
layout_b == library::LayoutTypeID::kTensorC64RSK64 &&
layout_c == library::LayoutTypeID::kTensorNC64HW64))) {
int interleave =
(layout_a == library::LayoutTypeID::kTensorNC32HW32) ? 32 : 64;
stride_activations.push_back(int(problem.w) * interleave);
stride_activations.push_back(int(problem.w) * int(problem.h) *
interleave);
stride_activations.push_back(int(problem.h) * int(problem.w) *
int(problem.c));
stride_filters.push_back(int(problem.k) * interleave);
stride_filters.push_back(int(problem.k) * int(problem.s) * interleave);
stride_filters.push_back(int(problem.k) * int(problem.s) *
int(problem.r) * interleave);
stride_output.push_back(int(problem.q) * interleave);
stride_output.push_back(int(problem.q) * int(problem.p) * interleave);
stride_output.push_back(int(problem.q) * int(problem.p) *
int(problem.k));
} else {
// Strides for the rest cases
stride_activations.push_back(int(problem.c));
stride_activations.push_back(int(problem.w) * int(problem.c));
stride_activations.push_back(int(problem.h) * int(problem.w) *
int(problem.c));
stride_filters.push_back(int(problem.c / problem.groups));
stride_filters.push_back(int(problem.s) * int(problem.c / problem.groups));
stride_filters.push_back(int(problem.r) * int(problem.s) *
int(problem.c / problem.groups));
stride_output.push_back(int(problem.k));
stride_output.push_back(int(problem.q) * int(problem.k));
stride_output.push_back(int(problem.q) * int(problem.p) *
int(problem.k));
}
switch (conv_kind) {
case library::ConvKind::kFprop:
configuration.stride_a = stride_activations;
configuration.stride_b = stride_filters;
configuration.stride_c = stride_output;
break;
case library::ConvKind::kDgrad:
configuration.stride_a = stride_output;
configuration.stride_b = stride_filters;
configuration.stride_c = stride_activations;
break;
case library::ConvKind::kWgrad:
configuration.stride_a = stride_output;
configuration.stride_b = stride_activations;
configuration.stride_c = stride_filters;
break;
default:
throw std::runtime_error(
"Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
};
protected:
//
// Data members
//
/// CONV problem obtained from problem space
Conv2dProblem problem_;
/// Device memory allocations
Conv2dWorkspace conv_workspace_;
/// CUTLASS parallel reduction operation to follow this* conv2d operation
library::Operation const *reduction_op_;
public:
//
// Methods
//
/// Ctor
Conv2dOperationProfiler(Options const &options);
/// Destructor
virtual ~Conv2dOperationProfiler();
Conv2dProblem const& problem() const { return problem_; }
/// Prints usage statement for the math function
virtual void print_usage(std::ostream &out) const;
/// Prints examples
virtual void print_examples(std::ostream &out) const;
/// Extracts the problem dimensions
virtual Status initialize_configuration(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Initializes workspace
virtual Status initialize_workspace(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Verifies CUTLASS against references
virtual bool verify_cutlass(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Measures performance results
virtual bool profile(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
protected:
/// Method to profile an initialized CUTLASS operation
virtual Status profile_cutlass_(
double &runtime,
Options const &options,
library::Operation const *operation,
void *arguments,
void *host_workspace,
void *device_workspace);
/// Initialize reduction problem dimensions and library::Operation
bool initialize_reduction_configuration_(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Initializes the performance result
void initialize_result_(
PerformanceResult &result,
Options const &options,
library::ConvDescription const &operation_desc,
ProblemSpace const &problem_space);
/// Verifies CUTLASS against host reference
bool verify_with_host_reference_(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Verifies CUTLASS against device reference
bool verify_with_device_reference_(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
#if CUTLASS_ENABLE_CUDNN
/// Verifies CUTLASS against cudnn reference
bool verify_with_cudnn_(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
#endif //#if CUTLASS_ENABLE_CUDNN
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////

449
tools/profiler/include/cutlass/profiler/conv3d_operation_profiler.h Normal file
View File

@@ -0,0 +1,449 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Defines profiling functionality for convolution
*/
#pragma once
#include <vector>
#include <string>
#include <memory>
#include <algorithm>
#include <unordered_map>
// CUTLASS Library includes
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
#include "cutlass/library/handle.h"
#include "cutlass/library/manifest.h"
#include "cutlass/library/singleton.h"
// Profiler includes
#include "options.h"
#include "device_context.h"
#include "operation_profiler.h"
#include "performance_result.h"
#include "problem_space.h"
#include "reduction_operation_profiler.h"
#if CUTLASS_ENABLE_CUDNN
#include "cudnn_helpers.h"
#endif //#if CUTLASS_ENABLE_CUDNN
#include "debug.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Abstract base class for each math function
class Conv3dOperationProfiler : public OperationProfiler {
public:
/// Problem structure obtained from problem space
struct Conv3dProblem {
int64_t n, d, h, w, c, z, p, q, k, t, r, s;
int64_t pad_d, pad_h, pad_w;
int64_t stride_d, stride_h, stride_w;
int64_t dilation_d, dilation_h, dilation_w;
std::vector<uint8_t> alpha;
std::vector<uint8_t> beta;
library::SplitKMode split_k_mode;
int64_t split_k_slices;
library::ConvModeID conv_mode;
library::Provider eq_gemm_provider;
// convolution with parallel interleaved reduction
// convolution epilogue (alpha, beta) = (1.0, 0.0)
// reduction epilogue (alpha, beta) = (Conv3dProblem::alpha, Conv3dProblem::beta)
std::vector<uint8_t> alpha_one;
std::vector<uint8_t> beta_zero;
//
// Methods
//
/// Total number of bytes loaded
int64_t bytes(library::ConvDescription const &operation_desc) const;
/// Total number of flops computed
int64_t flops(library::ConvDescription const &operation_desc) const;
/// Infers output size from the input size, padding, stride, and dilation
void set_default_output_size() {
z = ((d + pad_d - t * dilation_d) / stride_d) + 1;
p = ((h + pad_h - r * dilation_h) / stride_h) + 1;
q = ((w + pad_w - s * dilation_w) / stride_w) + 1;
}
// Returns equivalent gemm problem size for convolution
cutlass::gemm::GemmCoord eq_gemm_size(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return cutlass::gemm::GemmCoord(int(n * z * p * q), int(k), int(t * r * s * c));
case library::ConvKind::kDgrad: return cutlass::gemm::GemmCoord(int(n * d * h * w), int(c), int(t * r * s * k));
case library::ConvKind::kWgrad: return cutlass::gemm::GemmCoord(int(k), int(t * r * s * c), int(n * z * p * q));
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns extent for tensor A
std::vector<int> extent_a(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return {int(n), int(d), int(h), int(w), int(c)};
case library::ConvKind::kDgrad: return {int(n), int(z), int(p), int(q), int(k)};
case library::ConvKind::kWgrad: return {int(n), int(z), int(p), int(q), int(k)};
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns extent for tensor B
std::vector<int> extent_b(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return {int(k), int(t), int(r), int(s), int(c)};
case library::ConvKind::kDgrad: return {int(k), int(t), int(r), int(s), int(c)};
case library::ConvKind::kWgrad: return {int(n), int(d), int(h), int(w), int(c)};
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns extent for tensor C
std::vector<int> extent_c(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return {int(n), int(z), int(p), int(q), int(k)};
case library::ConvKind::kDgrad: return {int(n), int(d), int(h), int(w), int(c)};
case library::ConvKind::kWgrad: return {int(k), int(t), int(r), int(s), int(c)};
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns layout for equivalent gemm matrix A
library::LayoutTypeID eq_gemm_layout_a(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return library::LayoutTypeID::kRowMajor; // TN Gemm
case library::ConvKind::kDgrad: return library::LayoutTypeID::kRowMajor; // TT Gemm
case library::ConvKind::kWgrad: return library::LayoutTypeID::kColumnMajor; // NT Gemm
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns layout for equivalent gemm matrix B
library::LayoutTypeID eq_gemm_layout_b(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return library::LayoutTypeID::kColumnMajor; // TN Gemm
case library::ConvKind::kDgrad: return library::LayoutTypeID::kRowMajor; // TT Gemm
case library::ConvKind::kWgrad: return library::LayoutTypeID::kRowMajor; // NT Gemm
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns layout for equivalent gemm matrix C
library::LayoutTypeID eq_gemm_layout_c(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
// Gemm operator assumes column-major output
case library::ConvKind::kFprop:
case library::ConvKind::kDgrad:
case library::ConvKind::kWgrad: return library::LayoutTypeID::kColumnMajor;
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns leading dimension for equivalent gemm matrix A
int64_t eq_gemm_lda(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return eq_gemm_size(conv_kind).k();
case library::ConvKind::kDgrad: return eq_gemm_size(conv_kind).k();
case library::ConvKind::kWgrad: return eq_gemm_size(conv_kind).m();
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns leading dimension for equivalent gemm matrix B
int64_t eq_gemm_ldb(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return eq_gemm_size(conv_kind).k();
case library::ConvKind::kDgrad: return eq_gemm_size(conv_kind).n();
case library::ConvKind::kWgrad: return eq_gemm_size(conv_kind).n();
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns leading dimension for equivalent gemm matrix C
int64_t eq_gemm_ldc(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop:
case library::ConvKind::kDgrad:
case library::ConvKind::kWgrad: return eq_gemm_size(conv_kind).m();
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
};
/// Workspace used
struct Conv2dWorkspace {
/// Conv device allocations
DeviceAllocation *A;
DeviceAllocation *B;
DeviceAllocation *C;
DeviceAllocation *Computed;
DeviceAllocation *Reference;
/// Library configuration and arguments for convolution operator
library::Conv3dConfiguration configuration;
library::ConvArguments arguments;
/// Number of copies of the problem workspace which are visited sequentially during
/// profiling to avoid camping in the last level cache.
int problem_count;
/// Buffer used for the cutlass conv2d operations' host workspace
std::vector<uint8_t> host_workspace;
/// Buffer used for the cutlass operations' device workspace
DeviceAllocation device_workspace;
/// Library configuration and arguments for reduction operator
library::ReductionConfiguration reduction_configuration;
library::ReductionArguments reduction_arguments;
/// Buffer used for the cutlass reduction operations' host workspace
std::vector<uint8_t> reduction_host_workspace;
/// Host data buffers for host reference operation
/// host buffer for tensor
std::vector<uint8_t> host_tensor_a;
/// host buffer for tensor b
std::vector<uint8_t> host_tensor_b;
/// host buffer for tensor c
std::vector<uint8_t> host_tensor_c;
//
// Methods
//
Conv2dWorkspace():
A(nullptr), B(nullptr), C(nullptr), Computed(nullptr), Reference(nullptr) { }
// Returns stride vector for tensor A
std::vector<int64_t> stride_a(library::ConvKind const &conv_kind) {
return {
configuration.layout_a(conv_kind).stride()[0],
configuration.layout_a(conv_kind).stride()[1],
configuration.layout_a(conv_kind).stride()[2],
configuration.layout_a(conv_kind).stride()[3]
};
}
// Returns stride vector for tensor B
std::vector<int64_t> stride_b(library::ConvKind const &conv_kind) {
return {
configuration.layout_b(conv_kind).stride()[0],
configuration.layout_b(conv_kind).stride()[1],
configuration.layout_b(conv_kind).stride()[2],
configuration.layout_b(conv_kind).stride()[3]
};
}
// Returns stride vector for tensor C
std::vector<int64_t> stride_c(library::ConvKind const &conv_kind) {
return {
configuration.layout_c(conv_kind).stride()[0],
configuration.layout_c(conv_kind).stride()[1],
configuration.layout_c(conv_kind).stride()[2],
configuration.layout_c(conv_kind).stride()[3]
};
}
};
protected:
//
// Data members
//
/// CONV problem obtained from problem space
Conv3dProblem problem_;
/// Device memory allocations
Conv2dWorkspace conv_workspace_;
/// CUTLASS parallel reduction operation to follow this* conv2d operation
library::Operation const *reduction_op_;
public:
//
// Methods
//
/// Ctor
Conv3dOperationProfiler(Options const &options);
/// Destructor
virtual ~Conv3dOperationProfiler();
Conv3dProblem const& problem() const { return problem_; }
/// Prints usage statement for the math function
virtual void print_usage(std::ostream &out) const;
/// Prints examples
virtual void print_examples(std::ostream &out) const;
/// Extracts the problem dimensions
virtual Status initialize_configuration(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Initializes workspace
virtual Status initialize_workspace(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Verifies CUTLASS against references
virtual bool verify_cutlass(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Measures performance results
virtual bool profile(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
protected:
/// Updates the arguments structure for the CUTLASS operator based on
/// the problem index.
void set_cutlass_operator_arguments_(int problem_idx = 0);
/// Method to profile an initialized CUTLASS operation
virtual Status profile_cutlass_(
double &runtime,
Options const &options,
library::Operation const *operation,
void *arguments,
void *host_workspace,
void *device_workspace);
/// Initialize reduction problem dimensions and library::Operation
bool initialize_reduction_configuration_(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Initializes the performance result
void initialize_result_(
PerformanceResult &result,
Options const &options,
library::ConvDescription const &operation_desc,
ProblemSpace const &problem_space);
/// Verifies CUTLASS against host reference
bool verify_with_host_reference_(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Verifies CUTLASS against device reference
bool verify_with_device_reference_(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
#if CUTLASS_ENABLE_CUDNN
/// Verifies CUTLASS against cudnn reference
bool verify_with_cudnn_(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
#endif //#if CUTLASS_ENABLE_CUDNN
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////

358
tools/profiler/include/cutlass/profiler/cublas_helpers.h Normal file
View File

@@ -0,0 +1,358 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Helper functions for mapping CUTLASS concepts to cuBLAS.
*/
#pragma once
#if CUTLASS_ENABLE_CUBLAS
#include <cublas_v2.h>
#include "cutlass/cutlass.h"
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
#include "cutlass/blas3.h"
#include "options.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Converts a cuBLAS status to cutlass::Status
Status get_cutlass_status(cublasStatus_t cublas);
/// Converts a cuBLAS status to cutlass::profiler::Disposition
Disposition get_cutlass_disposition(cublasStatus_t cublas_status);
/// Maps a CUTLASS tensor layout to a cuBLAS transpose operation
bool get_cublas_transpose_operation(
cublasOperation_t &operation,
library::LayoutTypeID layout,
library::ComplexTransform transform = library::ComplexTransform::kNone);
/// Maps a CUTLASS numeric type to a cuBLAS data type enumeration
bool get_cublas_datatype(cublasDataType_t &data_type, library::NumericTypeID element_type);
/// Gets the cublas algorithm given threadblock tile dimensions and math opcode class
cublasGemmAlgo_t get_cublas_gemm_algo(
int cta_m,
int cta_n,
int cta_k,
library::OpcodeClassID opcode_class);
/// Returns a status if cuBLAS can satisfy a particular GEMM description
Status cublas_satisfies(library::GemmDescription const &desc);
/// Returns a status if cuBLAS can satisfy a particular RankK description
Status cublas_satisfies(library::RankKDescription const &desc);
/// Returns a status if cuBLAS can satisfy a particular TRMM description
Status cublas_satisfies(library::TrmmDescription const &desc);
/// Returns a status if cuBLAS can satisfy a particular SYMM/HEMM description
Status cublas_satisfies(library::SymmDescription const &desc);
/// This is a helper class to create cublasHandle_t automatically on CublasCreate object creation and
/// to destroy cublasHandle_t on CublasCreate object destruction.
/// Additionally, it provides implicit cast from CublasCreate's object to cublasHandle_t's object
class CublasCreate {
private:
cublasHandle_t handle;
cublasStatus_t status;
public:
CublasCreate() {
status = cublasCreate(&handle);
}
~CublasCreate() {
cublasDestroy(handle);
}
/// Implicit cast CublasCreate object to cublasHandle_t
operator cublasHandle_t() const { return handle; }
/// returns cublasStatus_t for handle creation
cublasStatus_t get_cublas_create_status() { return status; }
};
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace detail {
/// Selects one or more cuBLAS algorithms.
static void select_cublas_algorithms(
std::vector<cublasGemmAlgo_t> &algorithms,
Options const &options,
library::GemmDescription const &op_desc) {
library::OpcodeClassID const & opcode_class =
op_desc.tile_description.math_instruction.opcode_class;
switch (options.library.algorithm_mode) {
case AlgorithmMode::kMatching:
{
algorithms.push_back(get_cublas_gemm_algo(
op_desc.tile_description.threadblock_shape.m(),
op_desc.tile_description.threadblock_shape.n(),
op_desc.tile_description.threadblock_shape.k(),
opcode_class));
break;
}
case AlgorithmMode::kBest:
{
// Choose first enumerated mode. If none are enumerated, choose based on opcode class
// and evaluate all of them.
if (options.library.algorithms.empty()) {
// Enumerate all algorithms
if (opcode_class == library::OpcodeClassID::kSimt) {
for (int algo = CUBLAS_GEMM_DEFAULT;
algo <= CUBLAS_GEMM_ALGO23;
++algo) {
algorithms.push_back(cublasGemmAlgo_t(algo));
}
}
else {
for (int algo = CUBLAS_GEMM_DEFAULT_TENSOR_OP;
algo <= CUBLAS_GEMM_ALGO15_TENSOR_OP;
++algo) {
algorithms.push_back(cublasGemmAlgo_t(algo));
}
}
}
else {
// Use the listed algorithms
algorithms.reserve(options.library.algorithms.size());
for (int algo : options.library.algorithms) {
algorithms.push_back(reinterpret_cast<cublasGemmAlgo_t const &>(algo));
}
}
break;
}
case AlgorithmMode::kDefault:
{
// Use the library's default algorithm
algorithms.push_back((opcode_class == library::OpcodeClassID::kSimt ?
CUBLAS_GEMM_DEFAULT : CUBLAS_GEMM_DEFAULT_TENSOR_OP));
break;
}
default:
{
break;
}
}
}
/// Dispatcher to cublasGemmEx()
struct cublasGemmExDispatcher {
//
// Data members
//
library::GemmUniversalConfiguration configuration;
library::GemmUniversalArguments arguments;
// cublas-specific data structures to fill cublas API call arguments
cublasOperation_t trans_A;
cublasOperation_t trans_B;
cudaDataType_t data_type_A;
cudaDataType_t data_type_B;
cudaDataType_t data_type_C;
cudaDataType_t compute_data_type;
#if (__CUDACC_VER_MAJOR__ >= 11)
cublasComputeType_t compute_type;
#endif
cublasGemmAlgo_t algo;
Status status;
//
// Methods
//
cublasGemmExDispatcher(
library::GemmDescription const &op_desc,
library::GemmUniversalConfiguration configuration_,
library::GemmUniversalArguments arguments_,
cublasGemmAlgo_t algorithm = CUBLAS_GEMM_DFALT
);
/// Executes GEMM using these arguments
cublasStatus_t operator()(cublasHandle_t handle);
};
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Dispatcher to cublas rank k update kernels
struct cublasRankKDispatcher {
//
// Data members
//
library::RankKConfiguration configuration;
library::RankKArguments arguments;
// cublas-specific data structures to fill cublas API call arguments
cublasOperation_t trans_A;
cublasFillMode_t uplo;
cudaDataType_t data_type_A;
cudaDataType_t data_type_C;
cudaDataType_t compute_data_type;
#if (__CUDACC_VER_MAJOR__ >= 11)
cublasComputeType_t compute_type;
#endif
int num_ranks; //(rank-k or rank-2k)
BlasMode blas_mode; //(symmetric or hermitian)
Status status;
//
// Methods
//
cublasRankKDispatcher(
library::RankKDescription const &op_desc,
library::RankKConfiguration configuration_,
library::RankKArguments arguments_
);
/// Executes RankK using these arguments
cublasStatus_t operator()(cublasHandle_t handle);
};
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Dispatcher to cublasTrmm()
struct cublasTrmmDispatcher {
//
// Data members
//
library::TrmmConfiguration configuration;
library::TrmmArguments arguments;
// cublas-specific data structures to fill cublas API call arguments
cublasOperation_t trans_A;
cublasSideMode_t side;
cublasFillMode_t uplo;
cublasDiagType_t diag;
cudaDataType_t data_type_A;
cudaDataType_t data_type_B;
cudaDataType_t data_type_D;
cudaDataType_t compute_data_type;
#if (__CUDACC_VER_MAJOR__ >= 11)
cublasComputeType_t compute_type;
#endif
Status status;
//
// Methods
//
cublasTrmmDispatcher(
library::TrmmDescription const &op_desc,
library::TrmmConfiguration configuration_,
library::TrmmArguments arguments_
);
/// Executes TRMM using these arguments
cublasStatus_t operator()(cublasHandle_t handle);
};
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Dispatcher to cublas symm/hemm update kernels
struct cublasSymmDispatcher {
//
// Data members
//
library::SymmConfiguration configuration;
library::SymmArguments arguments;
// cublas-specific data structures to fill cublas API call arguments
cublasSideMode_t side;
cublasFillMode_t uplo;
cudaDataType_t data_type_A;
cudaDataType_t data_type_B;
cudaDataType_t data_type_C;
cudaDataType_t compute_data_type;
#if (__CUDACC_VER_MAJOR__ >= 11)
cublasComputeType_t compute_type;
#endif
BlasMode blas_mode; //(symmetric or hermitian)
Status status;
//
// Methods
//
cublasSymmDispatcher(
library::SymmDescription const &op_desc,
library::SymmConfiguration configuration_,
library::SymmArguments arguments_
);
/// Executes Symm using these arguments
cublasStatus_t operator()(cublasHandle_t handle);
};
///////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace detail
} // namespace profiler
} // namespace cutlass
#endif // #if CUTLASS_ENABLE_CUBLAS

590
tools/profiler/include/cutlass/profiler/cudnn_helpers.h Normal file
View File

@@ -0,0 +1,590 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Helper functions for mapping CUTLASS concepts to cuDNN.
*/
#pragma once
#if CUTLASS_ENABLE_CUDNN
#include <cuda_runtime.h>
#include <cudnn.h>
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/util/device_memory.h"
#include "cutlass/library/library.h"
#include "enumerated_types.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Converts a cuDNN status to cutlass::Status
Status get_cutlass_status(cudnnStatus_t cudnn_status);
/// Converts a cuDNN status to cutlass::profiler::Disposition
Disposition get_cutlass_disposition(cudnnStatus_t cudnn_status);
/// Checks cudnnStatus_t converts to cutlas status and returns if Status::kSuccess o.w. throws exception
Status checkCudnnErr(cudnnStatus_t cudnn_status);
/// Maps a CUTLASS conv mode to a cuDNN conv mode enumeration
bool get_cudnn_conv_mode(cudnnConvolutionMode_t &cudnn_conv_mode, conv::Mode conv_mode);
/// Maps a CUTLASS layout type to a cuDNN data type enumeration
bool get_cudnn_layout(cudnnTensorFormat_t &cudnn_layout, library::LayoutTypeID layout);
/// Maps a CUTLASS numeric type to a cuDNN data type enumeration
bool get_cudnn_datatype(cudnnDataType_t &cudnn_element_type, library::NumericTypeID element_type);
/// Maps CUTLASS math OpcodeClassID and MathOperationID to cuDNN math_type
bool get_cudnn_mathtype(cudnnMathType_t &cudnn_math_type, library::ConvDescription const &conv_desc);
/// Returns a status if cudnn can satisfy a particular Conv2d description
Status cudnn_satisfies(library::ConvDescription const &desc, library::Conv2dConfiguration const &configuration);
/// Returns a status if cudnn can satisfy a particular Conv3d description
Status cudnn_satisfies(library::ConvDescription const &desc, library::Conv3dConfiguration const &configuration);
/// Cudnn compute type seems to be hardcoded to float (To handle a possible cudnn issue)
float cast_cudnn_compute_type_to_float(library::NumericTypeID type, void const * src);
/// This is a helper class to create cudnnHandle_t automatically on CudnnCreate object creation and
/// to destroy cudnnHandle_t on CudnnCreate object destruction.
/// Additionally, it provides implicit cast from CudnnCreate's object to cudnnHandle_t's object
class CudnnCreate {
private:
cudnnHandle_t handle;
cudnnStatus_t status;
public:
CudnnCreate() {
status = cudnnCreate(&handle);
}
~CudnnCreate() {
cudnnDestroy(handle);
}
/// Implicit cast CudnnCreate object to cudnnHandle_t
operator cudnnHandle_t() const { return handle; }
/// returns cudnnStatus_t for handle creation
cudnnStatus_t get_cudnn_create_status() { return status; }
};
namespace detail {
/// Dispatcher to cudnn convolution operators
struct cudnnConvDispatcher {
//
// Data members
//
//library::Conv2dConfiguration configuration;
library::ConvArguments arguments;
library::ConvKind conv_kind;
// cudnn-specific data structures to fill cudnn API call arguments
// cudnn activation, filter, and output descriptors
cudnnTensorDescriptor_t activation_desc;
cudnnFilterDescriptor_t filter_desc;
cudnnTensorDescriptor_t output_desc;
cudnnConvolutionDescriptor_t conv_desc;
// cudnn datatypes
cudnnDataType_t data_type_activation;
cudnnDataType_t data_type_filter;
cudnnDataType_t data_type_output;
// cudnn layouts
cudnnTensorFormat_t layout_activation;
cudnnTensorFormat_t layout_filter;
cudnnTensorFormat_t layout_output;
// cudnn convolution mode
cudnnConvolutionMode_t conv_mode;
// cudnn math type (tensorop, tensorop with conversion, simt)
cudnnMathType_t math_type;
// cudnn compute data type
cudnnDataType_t compute_type;
// cudnn compute type seems to be hardcoded to float (to handle a possible a cudnn issue)
float alpha;
float beta;
// cudnn workspace
size_t workspace_size_in_bytes = 0;
cutlass::device_memory::allocation<char> workspace;
// select cudnn's implicit gemm precomputed algorithm with tensor operations
static cudnnConvolutionFwdAlgo_t const fprop_algo = CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM;
static cudnnConvolutionBwdDataAlgo_t const dgrad_algo = CUDNN_CONVOLUTION_BWD_DATA_ALGO_1;
static cudnnConvolutionBwdFilterAlgo_t const wgrad_algo = CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1;
Status status;
//
// Methods
//
// TODO: unify ctor cudnnConvDispatcher for conv2d and conv3d by unifying Conv2dConfiguration
// ctor for conv2d
cudnnConvDispatcher(
library::ConvDescription const &op_desc,
library::Conv2dConfiguration configuration,
library::ConvArguments arguments_,
cudnnHandle_t handle
):
//configuration(configuration_),
arguments(arguments_),
conv_kind(op_desc.conv_kind),
status(Status::kSuccess) {
bool good = true;
// Get cudnn datatype, layout, and convolution mode from library::ConvDescription
good = (good && get_cudnn_datatype(data_type_activation, op_desc.A.element));
good = (good && get_cudnn_datatype(data_type_filter, op_desc.B.element));
good = (good && get_cudnn_datatype(data_type_output, op_desc.C.element));
good = (good && get_cudnn_layout(layout_activation, op_desc.A.layout));
good = (good && get_cudnn_layout(layout_filter, op_desc.B.layout));
good = (good && get_cudnn_layout(layout_output, op_desc.C.layout));
good = (good && get_cudnn_conv_mode(conv_mode, configuration.problem_size.mode));
// Get cudnn mathtype (cudnnMathType_t)
good = (good && get_cudnn_mathtype(math_type, op_desc));
good = (good && get_cudnn_datatype(
compute_type,
op_desc.tile_description.math_instruction.element_accumulator));
// Check cutlass Conv2d description has equivalent operator in cudnn
if (!good) {
status = Status::kErrorNotSupported;
return;
}
// cudnn compute type seems to be hardcoded to float (to handle a possible a cudnn issue)
alpha = cast_cudnn_compute_type_to_float(op_desc.element_epilogue, arguments.alpha);
beta = cast_cudnn_compute_type_to_float(op_desc.element_epilogue, arguments.beta);
// Create convolution descriptor object
status = get_cutlass_status(cudnnCreateConvolutionDescriptor(&conv_desc));
// Configure convolution operator
std::vector<int> padding {configuration.problem_size.pad_h, configuration.problem_size.pad_w};
std::vector<int> stride {configuration.problem_size.stride_h, configuration.problem_size.stride_w};
std::vector<int> dilation {configuration.problem_size.dilation_h, configuration.problem_size.dilation_w};
status = get_cutlass_status(
cudnnSetConvolutionNdDescriptor(
conv_desc,
op_desc.conv_dim,
padding.data(),
stride.data(),
dilation.data(),
conv_mode,
compute_type
));
// Set groups
status = get_cutlass_status(cudnnSetConvolutionGroupCount(conv_desc, configuration.problem_size.groups));
// Create activation, filter, and output descriptor objects
status = get_cutlass_status(cudnnCreateTensorDescriptor(&activation_desc));
status = get_cutlass_status(cudnnCreateFilterDescriptor(&filter_desc));
status = get_cutlass_status(cudnnCreateTensorDescriptor(&output_desc));
// Set activation, filter, and output descriptor
status = get_cutlass_status(
cudnnSetTensor4dDescriptor(
activation_desc,
layout_activation,
data_type_activation,
configuration.problem_size.N,
configuration.problem_size.C,
configuration.problem_size.H,
configuration.problem_size.W
));
status = get_cutlass_status(
cudnnSetFilter4dDescriptor(
filter_desc,
data_type_filter,
layout_filter,
configuration.problem_size.K,
configuration.problem_size.C / configuration.problem_size.groups,
configuration.problem_size.R,
configuration.problem_size.S
));
status = get_cutlass_status(
cudnnSetTensor4dDescriptor(
output_desc,
layout_output,
data_type_output,
configuration.problem_size.N,
configuration.problem_size.K,
configuration.problem_size.P,
configuration.problem_size.Q
));
// Set math instruction to tensor op
status = get_cutlass_status(
cudnnSetConvolutionMathType(conv_desc, math_type));
// Initialize workspace
switch (conv_kind) {
case library::ConvKind::kFprop:
status = get_cutlass_status(
cudnnGetConvolutionForwardWorkspaceSize(
handle,
activation_desc,
filter_desc,
conv_desc,
output_desc,
fprop_algo,
&workspace_size_in_bytes
)); break;
case library::ConvKind::kDgrad:
status = get_cutlass_status(
cudnnGetConvolutionBackwardDataWorkspaceSize(
handle,
filter_desc,
output_desc,
conv_desc,
activation_desc,
dgrad_algo,
&workspace_size_in_bytes
)); break;
case library::ConvKind::kWgrad:
status = get_cutlass_status(
cudnnGetConvolutionBackwardFilterWorkspaceSize(
handle,
activation_desc,
output_desc,
conv_desc,
filter_desc,
wgrad_algo,
&workspace_size_in_bytes
)); break;
}
workspace = cutlass::device_memory::allocation<char>(workspace_size_in_bytes);
}
// ctor for conv3d
cudnnConvDispatcher(
library::ConvDescription const &op_desc,
library::Conv3dConfiguration configuration,
library::ConvArguments arguments_,
cudnnHandle_t handle
):
//configuration(configuration_),
arguments(arguments_),
conv_kind(op_desc.conv_kind),
status(Status::kSuccess) {
bool good = true;
// Get cudnn datatype, layout, and convolution mode from library::ConvDescription
good = (good && get_cudnn_datatype(data_type_activation, op_desc.A.element));
good = (good && get_cudnn_datatype(data_type_filter, op_desc.B.element));
good = (good && get_cudnn_datatype(data_type_output, op_desc.C.element));
good = (good && get_cudnn_layout(layout_activation, op_desc.A.layout));
good = (good && get_cudnn_layout(layout_filter, op_desc.B.layout));
good = (good && get_cudnn_layout(layout_output, op_desc.C.layout));
good = (good && get_cudnn_conv_mode(conv_mode, configuration.problem_size.mode));
// cudnn compute type seems to be hardcoded to float (to handle a possible a cudnn issue)
alpha = cast_cudnn_compute_type_to_float(op_desc.element_epilogue, arguments.alpha);
beta = cast_cudnn_compute_type_to_float(op_desc.element_epilogue, arguments.beta);
good = (good && get_cudnn_datatype(
compute_type,
op_desc.tile_description.math_instruction.element_accumulator));
// Check cutlass Conv2d description has equivalent operator in cudnn
if (!good) {
status = Status::kErrorNotSupported;
}
// Create convolution descriptor object
status = get_cutlass_status(cudnnCreateConvolutionDescriptor(&conv_desc));
// Configure convolution operator
std::vector<int> padding {configuration.problem_size.pad_d, configuration.problem_size.pad_h, configuration.problem_size.pad_w};
std::vector<int> stride {configuration.problem_size.stride_d, configuration.problem_size.stride_h, configuration.problem_size.stride_w};
std::vector<int> dilation {configuration.problem_size.dilation_d, configuration.problem_size.dilation_h, configuration.problem_size.dilation_w};
status = get_cutlass_status(
cudnnSetConvolutionNdDescriptor(
conv_desc,
op_desc.conv_dim,
padding.data(),
stride.data(),
dilation.data(),
conv_mode,
compute_type
));
// Set groups
status = get_cutlass_status(cudnnSetConvolutionGroupCount(conv_desc, configuration.problem_size.groups));
// Create activation, filter, and output descriptor objects
status = get_cutlass_status(cudnnCreateTensorDescriptor(&activation_desc));
status = get_cutlass_status(cudnnCreateFilterDescriptor(&filter_desc));
status = get_cutlass_status(cudnnCreateTensorDescriptor(&output_desc));
// Set activation descriptor
std::vector<int> activation_extent {
configuration.problem_size.N,
configuration.problem_size.C,
configuration.problem_size.D,
configuration.problem_size.H,
configuration.problem_size.W
};
std::vector<int> activation_stride {
configuration.layout_activations.stride()[3],
1,
configuration.layout_activations.stride()[2],
configuration.layout_activations.stride()[1],
configuration.layout_activations.stride()[0]
};
status = get_cutlass_status(
cudnnSetTensorNdDescriptor(
activation_desc,
data_type_activation,
op_desc.conv_dim + 2,
activation_extent.data(),
activation_stride.data()
));
// Set filter descriptor
std::vector<int> filter_extent {
configuration.problem_size.K,
configuration.problem_size.C,
configuration.problem_size.T,
configuration.problem_size.R,
configuration.problem_size.S
};
std::vector<int> filter_stride {
configuration.layout_filters.stride()[3],
1,
configuration.layout_filters.stride()[2],
configuration.layout_filters.stride()[1],
configuration.layout_filters.stride()[0]
};
status = get_cutlass_status(
cudnnSetFilterNdDescriptor(
filter_desc,
data_type_filter,
layout_filter,
op_desc.conv_dim + 2,
filter_extent.data()
));
// Set output descriptor
std::vector<int> output_extent {
configuration.problem_size.N,
configuration.problem_size.K,
configuration.problem_size.Z,
configuration.problem_size.P,
configuration.problem_size.Q
};
std::vector<int> output_stride {
configuration.layout_output.stride()[3],
1,
configuration.layout_output.stride()[2],
configuration.layout_output.stride()[1],
configuration.layout_output.stride()[0]
};
status = get_cutlass_status(
cudnnSetTensorNdDescriptor(
output_desc,
data_type_output,
op_desc.conv_dim + 2,
output_extent.data(),
output_stride.data()
));
// Set math instruction to tensor op
status = get_cutlass_status(
cudnnSetConvolutionMathType(conv_desc, math_type));
// Initialize workspace
switch (conv_kind) {
case library::ConvKind::kFprop:
status = get_cutlass_status(
cudnnGetConvolutionForwardWorkspaceSize(
handle,
activation_desc,
filter_desc,
conv_desc,
output_desc,
fprop_algo,
&workspace_size_in_bytes
)); break;
case library::ConvKind::kDgrad:
status = get_cutlass_status(
cudnnGetConvolutionBackwardDataWorkspaceSize(
handle,
filter_desc,
output_desc,
conv_desc,
activation_desc,
dgrad_algo,
&workspace_size_in_bytes
)); break;
case library::ConvKind::kWgrad:
status = get_cutlass_status(
cudnnGetConvolutionBackwardFilterWorkspaceSize(
handle,
activation_desc,
output_desc,
conv_desc,
filter_desc,
wgrad_algo,
&workspace_size_in_bytes
)); break;
}
workspace = cutlass::device_memory::allocation<char>(workspace_size_in_bytes);
}
/// Executes Conv2d operator from cudnn library
cudnnStatus_t operator()(cudnnHandle_t handle) {
switch (conv_kind) {
case library::ConvKind::kFprop:
return cudnnConvolutionForward(
handle,
&alpha,
activation_desc,
activation(),
filter_desc,
filter(),
conv_desc,
fprop_algo,
workspace.get(),
workspace_size_in_bytes,
&beta,
output_desc,
arguments.D
);
case library::ConvKind::kDgrad:
return cudnnConvolutionBackwardData(
handle,
&alpha,
filter_desc,
filter(),
output_desc,
output(),
conv_desc,
dgrad_algo,
workspace.get(),
workspace_size_in_bytes,
&beta,
activation_desc,
arguments.D
);
case library::ConvKind::kWgrad:
return cudnnConvolutionBackwardFilter(
handle,
&alpha,
activation_desc,
activation(),
output_desc,
output(),
conv_desc,
wgrad_algo,
workspace.get(),
workspace_size_in_bytes,
&beta,
filter_desc,
arguments.D
);
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns Activation Tensor
void const * activation() const {
switch(conv_kind) {
case library::ConvKind::kFprop : return arguments.A;
case library::ConvKind::kDgrad : return arguments.C;
case library::ConvKind::kWgrad : return arguments.B;
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns Filter Tensor
void const *filter() const {
switch(conv_kind) {
case library::ConvKind::kFprop : return arguments.B;
case library::ConvKind::kDgrad : return arguments.B;
case library::ConvKind::kWgrad : return arguments.C;
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// Returns Output Tensor
void const *output() const {
switch(conv_kind) {
case library::ConvKind::kFprop : return arguments.C;
case library::ConvKind::kDgrad : return arguments.A;
case library::ConvKind::kWgrad : return arguments.A;
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
};
} // namespace detail
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif //#if CUTLASS_ENABLE_CUDNN
} // namespace profiler
} // namespace cutlass

96
tools/profiler/include/cutlass/profiler/cutlass_profiler.h Normal file
View File

@@ -0,0 +1,96 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Execution environment
*/
#pragma once
// CUTLASS Library includes
#include "cutlass/library/library.h"
#include "cutlass/library/manifest.h"
#include "cutlass/library/singleton.h"
#include "options.h"
#include "operation_profiler.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// CUTLASS Profiler application
class CutlassProfiler {
private:
//
// Data members
//
/// Performance testbench options
Options options_;
/// Entry points for each operation
OperationProfilerVector operation_profilers_;
private:
/// Prints usage
void print_usage_(std::ostream &);
/// Prints usage
void print_options_(std::ostream &);
/// Initializes the device
void initialize_device_();
/// Enumerates all operations
void enumerate_();
/// Profiles all operations
int profile_();
public:
CutlassProfiler(Options const &options);
~CutlassProfiler();
/// Invokes profiling operations
int operator()();
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////

56
tools/profiler/include/cutlass/profiler/debug.h Normal file
View File

@@ -0,0 +1,56 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief
*/
#pragma once
#include <iostream>
//#define report(x) { std::cout << "\033[31m" << __FILE__ << ":" << __LINE__ << " " << x << "\033[0m" << std::endl; }
//#define report(x) {}
// Enable/Disable Profiler debug prints
//#define DEBUG_PROFILER
//RED 31m // profiler prints debug messages in red
//YELLOW 33m // ir prints debug messages in yellow
#ifndef DEBUG_PROFILER
#define debugprof(...)
#else
#define debugprof(...) do { \
printf("\033[33m[DEBUG PROF] %s:%d | ", __FILE__, __LINE__); \
printf(__VA_ARGS__); \
printf("\033[0m\n"); \
} while (0)
#endif

232
tools/profiler/include/cutlass/profiler/device_allocation.h Normal file
View File

@@ -0,0 +1,232 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Execution environment
*/
#pragma once
#include <stdexcept>
#include <list>
#include <vector>
#include "cutlass/library/library.h"
#include "cutlass/util/distribution.h"
#include "enumerated_types.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Device memory allocation
class DeviceAllocation {
private:
/// Data type of contained elements
library::NumericTypeID type_;
/// Gets the stride between elements
size_t batch_stride_;
/// Capacity in elements of device allocation
size_t capacity_;
/// Pointer to device memory
void *pointer_;
/// Layout type ID
library::LayoutTypeID layout_;
/// Stride vector
std::vector<int64_t> stride_;
/// Extent vector
std::vector<int> extent_;
/// Support allocating a 'batch' of non-overlapping tensors in contiguous memory
int batch_count_;
/// Buffer holding TensorRef instance to recently allocated memory
std::vector<uint8_t> tensor_ref_buffer_;
public:
//
// Static member functions
//
/// Determines the number of bytes needed to represent this numeric type
static size_t bytes(library::NumericTypeID type, size_t capacity);
/// Returns the stride of a packed layout
static std::vector<int64_t> get_packed_layout(
library::LayoutTypeID layout_id,
std::vector<int> const &extent);
/// returns the capacity needed
static size_t construct_layout(
void *bytes,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
std::vector<int64_t> &stride);
/// Returns true if two blocks have exactly the same value
static bool block_compare_equal(
library::NumericTypeID numeric_type,
void const *ptr_A,
void const *ptr_B,
size_t capacity);
/// Returns true if two blocks have approximately the same value
static bool block_compare_relatively_equal(
library::NumericTypeID numeric_type,
void const *ptr_A,
void const *ptr_B,
size_t capacity,
double epsilon,
double nonzero_floor);
public:
//
// Methods
//
DeviceAllocation();
DeviceAllocation(library::NumericTypeID type, size_t capacity);
DeviceAllocation(
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
std::vector<int64_t> const &stride = std::vector<int64_t>(),
int batch_count = 1);
~DeviceAllocation();
DeviceAllocation &reset();
/// Allocates device memory of a given type and capacity
DeviceAllocation &reset(library::NumericTypeID type, size_t capacity);
/// Allocates memory for a given layout and tensor
DeviceAllocation &reset(
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
std::vector<int64_t> const &stride = std::vector<int64_t>(),
int batch_count = 1);
/// Returns a buffer owning the tensor reference
std::vector<uint8_t> &tensor_ref() {
return tensor_ref_buffer_;
}
bool good() const;
/// Data type of contained elements
library::NumericTypeID type() const;
/// Pointer to start of device memory allocation
void *data() const;
/// Pointer to the first element of a batch
void *batch_data(int batch_idx) const;
/// Gets the layout type
library::LayoutTypeID layout() const;
/// Gets the stride vector
std::vector<int64_t> const & stride() const;
/// Gets the extent vector
std::vector<int> const & extent() const;
/// Gets the number of adjacent tensors in memory
int batch_count() const;
/// Gets the stride (in units of elements) between items
int64_t batch_stride() const;
/// Gets the stride (in units of bytes) between items
int64_t batch_stride_bytes() const;
/// Capacity of allocation in number of elements
size_t capacity() const;
/// Capacity of allocation in bytes
size_t bytes() const;
/// Initializes a device allocation to a random distribution using cuRAND
void initialize_random_device(int seed, Distribution dist);
/// Initializes a host allocation to a random distribution using std::cout
void initialize_random_host(int seed, Distribution dist);
/// Initializes a device allocation to a sequential distribution
void initialize_sequential_device(Distribution dist);
/// Initializes a host allocation to a sequential distribution
void initialize_sequential_host(Distribution dist);
/// Initializes a device allocation to a random distribution using cuRAND
void initialize_random_sparsemeta_device(int seed, int MetaSizeInBits);
/// Initializes a host allocation to a random distribution using std::cout
void initialize_random_sparsemeta_host(int seed, int MetaSizeInBits);
/// Uniformly fills a tensor with a value when provided o.w. zero
void fill(double value);
/// Copies from an equivalent-sized tensor in device memory
void copy_from_device(void const *ptr);
/// Copies from an equivalent-sized tensor in device memory
void copy_from_host(void const *ptr);
/// Copies from an equivalent-sized tensor in device memory
void copy_to_host(void *ptr);
/// Writes a tensor to csv
void write_tensor_csv(std::ostream &out);
};
using DeviceAllocationList = std::list<DeviceAllocation>;
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////

130
tools/profiler/include/cutlass/profiler/device_context.h Normal file
View File

@@ -0,0 +1,130 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief
*/
#pragma once
#include <map>
#include <string>
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
#include "options.h"
#include "device_allocation.h"
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Collection of allocations on the device
class DeviceContext {
public:
//
// Type definitions
//
using AllocationMap = std::map<std::string, DeviceAllocation *>;
private:
//
// Data members
//
/// Memory allocations that exist (owning)
DeviceAllocationList device_memory_;
/// Non-owning set of named allocations
AllocationMap allocations_;
public:
/// Allocates memory of a given type, capacity (elements), and name
DeviceAllocation *allocate_block(
std::string const &name,
library::NumericTypeID type,
size_t capacity);
/// Allocates memory of a given type, capacity (elements), and name
DeviceAllocation *allocate_tensor(
std::string const &name,
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
std::vector<int64_t> const &stride = std::vector<int64_t>(),
int batch_count = 1);
/// Allocates memory of a given type, capacity (elements), and name
DeviceAllocation *allocate_tensor(
Options const &options,
std::string const &name,
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
std::vector<int64_t> const &stride,
int batch_count,
int seed_shift = 0);
/// Allocates memory for sparse meta data
DeviceAllocation *allocate_sparsemeta_tensor(
Options const &options,
std::string const &name,
library::NumericTypeID type,
library::LayoutTypeID layout_id,
library::NumericTypeID type_a,
std::vector<int> const &extent,
std::vector<int64_t> const &stride,
int batch_count,
int seed_shift = 0);
/// Clears named allocations (but does not necessarily free memory)
void clear();
/// Frees all device memory allocations
void free();
/// Gets the allocation by name
DeviceAllocation &at(std::string const &name);
size_t size() const;
AllocationMap::iterator begin();
AllocationMap::iterator end();
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass

169
tools/profiler/include/cutlass/profiler/enumerated_types.h Normal file
View File

@@ -0,0 +1,169 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Provides several functions for filling tensors with data.
*/
#pragma once
#include <string>
#include <vector>
#include <map>
#include <iostream>
#include "cutlass/library/library.h"
#define TRACE(x) { std::cout << __FILE__ << ":" << __LINE__ << " " << x << std::endl; }
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename T>
T from_string(std::string const &);
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Enumerated type describing how the performance testbench evaluates kernels.
enum class ExecutionMode {
kProfile, ///< regular verification and profiling
kDryRun, ///< no kernels are launched or workspaces allocated; used to assess what operators might be launched
kEnumerate, ///< no kernels launched or workspaces allocated; lists all operation kind and operations
kTrace, ///< executes a single device-side computation with no other kernel launches
kInvalid
};
/// Converts a ExecutionMode enumerant to a string
char const *to_string(ExecutionMode mode, bool pretty = false);
/// Parses a ExecutionMode enumerant from a string
template <>
ExecutionMode from_string<ExecutionMode>(std::string const &str);
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Library algorithm mode
enum class AlgorithmMode {
kMatching, ///< compare against best matching algorithm
kBest, ///< evaluate all library algorithms and report best
kDefault, ///< use the library's default algorithm option
kInvalid
};
/// Converts a ExecutionMode enumerant to a string
char const *to_string(AlgorithmMode mode, bool pretty = false);
/// Parses a ExecutionMode enumerant from a string
template <>
AlgorithmMode from_string<AlgorithmMode>(std::string const &str);
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Outcome of a performance test
enum class Disposition {
kPassed,
kFailed,
kNotRun,
kIncorrect,
kNotVerified,
kInvalidProblem,
kNotSupported,
kInvalid
};
/// Converts a Disposition enumerant to a string
char const *to_string(Disposition disposition, bool pretty = false);
/// Parses a Disposition enumerant from a string
template <>
Disposition from_string<Disposition>(std::string const &str);
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Indicates when to save
enum class SaveWorkspace {
kNever,
kIncorrect,
kAlways,
kInvalid
};
/// Converts a SaveWorkspace enumerant to a string
char const *to_string(SaveWorkspace save_option, bool pretty = false);
/// Parses a SaveWorkspace enumerant from a string
template <>
SaveWorkspace from_string<SaveWorkspace>(std::string const &str);
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Indicates the type of kernel argument
// ArgumentType can be both ScalarType or NumericType. Thus, enums kScalar and kNumeric
// 1) kScalar: e.g. of a Scalar ArgumentType is u32 is a Scalar type.
// Its c++ equivalent as "type name = initializer" is "u32 m = 32"
// 2) kNumeric: e.g. of a Numeric ArgumentType is NumericTypeID is a Numeric type.
// Its c++ equivalent as "type name = initializer" is "NumericTypeID numeric_type = u32"
enum class ArgumentTypeID {
kScalar,
kInteger,
kTensor,
kBatchedTensor,
kStructure,
kEnumerated,
kInvalid
};
/// Converts a ArgumentTypeID enumerant to a string
char const *to_string(ArgumentTypeID type, bool pretty = false);
/// Parses a ArgumentTypeID enumerant from a string
template <>
ArgumentTypeID from_string<ArgumentTypeID>(std::string const &str);
/////////////////////////////////////////////////////////////////////////////////////////////////
// Profiler typedefs
using ProviderVector = std::vector<library::Provider>;
using DispositionMap = std::map<library::Provider, Disposition>;
/////////////////////////////////////////////////////////////////////////////////////////////////
// Print vector for the report
template <typename T>
std::ostream& operator<< (std::ostream& out, const std::vector<T>& v) {
for(int i = 0; i < v.size(); ++i) {
out << to_string(v[i], true) << (i+1 != v.size() ? "," : "");
}
return out;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass

275
tools/profiler/include/cutlass/profiler/gemm_operation_profiler.h Normal file
View File

@@ -0,0 +1,275 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Defines a math function
*/
#pragma once
#include <vector>
#include <string>
#include <memory>
#include <algorithm>
#include <unordered_map>
// CUTLASS Library includes
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
#include "cutlass/library/manifest.h"
// Profiler includes
#include "options.h"
#include "device_context.h"
#include "operation_profiler.h"
#include "performance_result.h"
#include "problem_space.h"
#include "reduction_operation_profiler.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Abstract base class for each math function
class GemmOperationProfiler : public OperationProfiler {
public:
/// Problem structure obtained from problem space
struct GemmProblem {
cutlass::library::GemmUniversalMode mode;
int64_t m;
int64_t n;
int64_t k;
int64_t lda;
int64_t ldb;
int64_t ldc;
std::vector<uint8_t> alpha;
std::vector<uint8_t> beta;
cutlass::library::SplitKMode split_k_mode;
int split_k_slices;
int batch_count;
cutlass::library::RasterOrder raster_order;
// gemm with parallel interleaved reduction
// gemm epilogue (alpha, beta) = (1.0, 0.0)
// reduction epilogue (alpha, beta) = (GemmProblem::alpha, GemmProblem::beta)
std::vector<uint8_t> alpha_one;
std::vector<uint8_t> beta_zero;
//
// Methods
//
GemmProblem():
mode(library::GemmUniversalMode::kGemm),
m(16), n(16), k(16), lda(0), ldb(0), ldc(0), split_k_slices(1), batch_count(1),
raster_order(cutlass::library::RasterOrder::kHeuristic){ }
/// Parses the problem
Status parse(
library::GemmDescription const &operation_desc,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Total number of bytes loaded
int64_t bytes(library::GemmDescription const &operation_desc) const;
/// Total number of flops computed
int64_t flops(library::GemmDescription const &operation_desc) const;
/// Initializes a performance result
void initialize_result(
PerformanceResult &result,
library::GemmDescription const &operation_desc,
ProblemSpace const &problem_space);
};
/// Workspace used
struct GemmWorkspace {
DeviceAllocation *A;
DeviceAllocation *B;
DeviceAllocation *C;
DeviceAllocation *Computed;
DeviceAllocation *Reference;
/// Number of copies of the problem workspace which are visited sequentially during
/// profiling to avoid camping in the last level cache.
int problem_count;
library::GemmUniversalConfiguration configuration;
library::GemmUniversalArguments arguments;
/// Buffer used for the operation's host workspace
std::vector<uint8_t> host_workspace;
/// Buffer used for the operations' device workspace
DeviceAllocation device_workspace;
/// Library configuration and arguments for reduction operator
library::ReductionConfiguration reduction_configuration;
library::ReductionArguments reduction_arguments;
/// Buffer used for the cutlass reduction operations' host workspace
std::vector<uint8_t> reduction_host_workspace;
//
// Methods
//
GemmWorkspace():
A(nullptr), B(nullptr), C(nullptr), Computed(nullptr), Reference(nullptr), problem_count(1) { }
};
protected:
//
// Data members
//
/// GEMM problem obtained from problem space
GemmProblem problem_;
/// Device memory allocations
GemmWorkspace gemm_workspace_;
/// CUTLASS parallel reduction operation to follow this* gemm operation
library::Operation const *reduction_op_;
public:
//
// Methods
//
/// Ctor
GemmOperationProfiler(Options const &options);
/// Destructor
virtual ~GemmOperationProfiler();
GemmProblem const& problem() const { return problem_; }
/// Prints usage statement for the math function
virtual void print_usage(std::ostream &out) const;
/// Prints examples
virtual void print_examples(std::ostream &out) const;
/// Extracts the problem dimensions
virtual Status initialize_configuration(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Initializes workspace
virtual Status initialize_workspace(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Verifies CUTLASS against references
virtual bool verify_cutlass(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Measures performance results
virtual bool profile(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
protected:
/// Initializes the performance result
void initialize_result_(
PerformanceResult &result,
Options const &options,
library::GemmDescription const &operation_desc,
ProblemSpace const &problem_space);
/// Verifies CUTLASS against references
bool verify_with_cublas_(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Verifies CUTLASS against host and device references
bool verify_with_reference_(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Method to profile a CUTLASS Operation
Status profile_cutlass_(
double &runtime,
Options const &options,
library::Operation const *operation,
void *arguments,
void *host_workspace,
void *device_workspace);
/// Initialize reduction problem dimensions and library::Operation
bool initialize_reduction_configuration_(
library::Operation const *operation,
ProblemSpace::Problem const &problem);
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////

72
tools/profiler/include/cutlass/profiler/gpu_timer.h Normal file
View File

@@ -0,0 +1,72 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Defines a math function
*/
#pragma once
#include <cuda_runtime.h>
#include "cutlass/cutlass.h"
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
struct GpuTimer {
cudaEvent_t events[2];
//
// Methods
//
GpuTimer();
~GpuTimer();
/// Records a start event in the stream
void start(cudaStream_t stream = nullptr);
/// Records a stop event in the stream
void stop(cudaStream_t stream = nullptr);
/// Records a stop event in the stream and synchronizes on the stream
void stop_and_wait(cudaStream_t stream = nullptr);
/// Returns the duration in milliseconds
double duration(int iterations = 1) const;
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass

259
tools/profiler/include/cutlass/profiler/operation_profiler.h Normal file
View File

@@ -0,0 +1,259 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Defines a math function
*/
#pragma once
#include <vector>
#include <string>
#include <memory>
#include <unordered_map>
// CUTLASS includes
#include "cutlass/trace.h"
// CUTLASS Library includes
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
#include "cutlass/library/manifest.h"
// Profiler includes
#include "options.h"
#include "device_context.h"
#include "performance_result.h"
#include "performance_report.h"
#include "problem_space.h"
#include "debug.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Abstract base class for each math function
class OperationProfiler {
public:
protected:
//
// Data members
//
/// Top-level operation kind
library::OperationKind kind_;
/// Human readable description
std::string description_;
/// Arguments parsed from command line
ArgumentDescriptionVector arguments_;
/// List of providers used to verify and compare each result
ProviderVector verification_providers_;
/// Model performance result initialized by the operation profiler with workload statistics
/// and reasonable default state.
PerformanceResult model_result_;
/// Performance result vector constructed by profiling the operation
PerformanceResultVector results_;
public:
//
// Methods
//
/// Ctor
OperationProfiler();
OperationProfiler(
Options const &options,
library::OperationKind kind,
ArgumentDescriptionVector const &arguments = ArgumentDescriptionVector(),
ProviderVector const & verification_providers = ProviderVector());
/// Destructor
virtual ~OperationProfiler();
/// Obtains the operation kind
library::OperationKind kind() const { return kind_; }
/// Gets the schema description
std::string const &description() const;
/// Returns a reference to the arguments
ArgumentDescriptionVector const &arguments() const { return arguments_; }
public:
//
// Basic overrides
//
/// Prints usage statement for the math function
virtual void print_usage(std::ostream &out) const;
/// Prints examples
virtual void print_examples(std::ostream &out) const =0;
/// Entry point to profile all operations in the manifest
virtual int profile_all(
Options const &options,
library::Manifest const &manifest,
DeviceContext &device_context);
public:
//
// Operation-specific phases of verification and profiling
//
/// Extracts the problem dimensions
virtual Status initialize_configuration(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) = 0;
/// Initializes workspace
virtual Status initialize_workspace(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) = 0;
/// Verifies CUTLASS against references
virtual bool verify_cutlass(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) = 0;
/// Measures performance results
virtual bool profile(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) = 0;
public:
//
// Static helpers
//
/// Sleep for a given duration in ms
static void sleep(int sleep_duration);
/// Returns true if the current operation description satisfies the problem space
static bool satisfies(
library::OperationDescription const &op_desc,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Compares tensors for equality
static Disposition compare_tensors(
Options const &options,
DeviceAllocation &experimental,
DeviceAllocation &reference,
int64_t count = 0);
static void save_workspace(
DeviceContext &device_context,
Options const &options,
library::OperationDescription const &desc,
library::Provider provider,
library::Provider verification_provider = library::Provider::kInvalid);
/// Helper to set a performance result member
static void set_argument(
PerformanceResult &result,
char const *name,
ProblemSpace const &problem_space,
std::string const &value);
/// Helper to set a performance result member
static void set_argument(
PerformanceResult &result,
char const *name,
ProblemSpace const &problem_space,
int64_t value);
protected:
/// Sets operation description
static void initialize_result_(
PerformanceResult &result,
library::OperationDescription const &operation_desc,
ProblemSpace const &problem_space);
/// Method to profile an initialized CUTLASS operation
virtual Status profile_cutlass_(
double &runtime,
Options const &options,
library::Operation const *operation,
void *arguments,
void *host_workspace,
void *device_workspace);
private:
/// finds string matches filter_string in operation_name
bool find_string_matches_(
std::string const &filter_string,
std::string const &operation_name);
};
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Vector of owning operation profilers
using OperationProfilerVector = std::vector<std::unique_ptr<OperationProfiler>>;
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////

334
tools/profiler/include/cutlass/profiler/options.h Normal file
View File

@@ -0,0 +1,334 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Command line options for performance test program
*/
#pragma once
#include <string>
#include <vector>
#include <map>
#include <cuda_runtime.h>
#include "cutlass/util/command_line.h"
#include "cutlass/util/distribution.h"
#include "cutlass/library/library.h"
#include "enumerated_types.h"
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Global options
class Options {
public:
/// Cublas and cuDNN options
struct Library {
//
// Data members
//
/// Algorithm mode
AlgorithmMode algorithm_mode;
/// Algorithm enumerants
std::vector<int> algorithms;
//
// Methods
//
Library(CommandLine const &cmdline);
void print_usage(std::ostream &out) const;
void print_options(std::ostream &out, int indent = 0) const;
};
/// Options related to the selected device
struct Device {
/// Device ID
int device;
/// CUDA Device properties
cudaDeviceProp properties;
/// Total memory allocation on device
size_t maximum_capacity;
//
// Methods
//
Device(CommandLine const &cmdline);
void print_usage(std::ostream &out) const;
void print_options(std::ostream &out, int indent = 0) const;
void print_device_info(std::ostream &out) const;
/// Returns the compute capability of the listed device (e.g. 61, 60, 70, 75)
int compute_capability() const;
};
/// Options related to initializing input tensors
struct Initialization {
/// If true, data is initialized randomly. If false, no initialization is performed after
/// allocating tensors.
bool enabled;
/// If true, data distribution is set by the user and is not allowed to change
/// If false, data distribution is allowed to change based on element_type (library::NumericTypeID)
bool fix_data_distribution;
/// Data distribution for input tensors
Distribution data_distribution;
/// Source of random tensor elements
library::Provider provider;
/// Random number generator seed.
int seed;
//
// Methods
//
Initialization(CommandLine const &cmdline);
void print_usage(std::ostream &out) const;
void print_options(std::ostream &out, int indent = 0) const;
/// Helper to parse a Distribution object from the command line parser
static void get_distribution(
cutlass::CommandLine const &args,
std::string const &arg,
cutlass::Distribution &dist);
};
/// Options related to verification of the result
struct Verification {
//
// Data members
//
/// If true, kernels are verified before they are profiled
bool enabled;
/// If true, causes profiler to return an error code if no reference check is run.
/// Only valid when verification is enabled.
bool required;
/// Relative error threshold - zero to require bit-level consistency
double epsilon;
/// Values smaller than this are assumed to be zero
double nonzero_floor;
/// List of providers used to verify each result
ProviderVector providers;
/// Indicates when to save the workspace
SaveWorkspace save_workspace;
//
// Methods
//
Verification(CommandLine const &cmdline);
void print_usage(std::ostream &out) const;
void print_options(std::ostream &out, int indent = 0) const;
/// Returns true if a provider is enabled
bool provider_enabled(library::Provider provider) const;
/// Returns the index of a provider if its enabled
size_t index(library::Provider provider) const;
};
/// Options related to profiling
struct Profiling {
/// Number of workspaces to rotate through to avoid cache-resident working sets
int workspace_count;
/// Number of iterations to warmup each kernel prior to profiling
int warmup_iterations;
/// Number of iterations to profile each kernel - if 0, kernels are launched up to the profiling duration
int iterations;
/// Number of ms to sleep between profiling periods (ms)
int sleep_duration;
/// If true, profiling is actually conducted.
bool enabled;
/// If true, profiling returns an error code if no kernels are found to match the filters.
bool error_on_no_match = false;
/// List of providers of each functionality to be profiled
ProviderVector providers;
//
// Methods
//
Profiling(CommandLine const &cmdline);
void print_usage(std::ostream &out) const;
void print_options(std::ostream &out, int indent = 0) const;
/// Returns true if a provider is enabled
bool provider_enabled(library::Provider provider) const;
/// Returns the index of a provider if its enabled
size_t index(library::Provider provider) const;
};
/// Options related to reporting
struct Report {
/// If true, result is appended to possibly existing file
bool append;
/// Path to a file containing results
std::string output_path;
/// Path to a file containing junit xml results
std::string junit_output_path;
/// Sequence of tags to attach to each result
std::vector<std::pair<std::string, std::string>> pivot_tags;
/// If true, reports status of all kernels including those that were
/// not run for the given arguments
bool report_not_run;
/// Prints human-readable text to stdout. If false, nothing is written to stdout
bool verbose;
/// Sort results by (currently by flops-per-byte)
bool sort_results;
/// Prints the name of the kernel being profiled before running the kernel.
/// This is useful for determining which kernel is causing a run of the profiler to hang
bool print_kernel_before_running;
//
// Methods
//
Report(CommandLine const &cmdline);
void print_usage(std::ostream &out) const;
void print_options(std::ostream &out, int indent = 0) const;
};
/// Options related to printing usage and version information
struct About {
/// If true, usage is printed and the program ends.
bool help;
/// Prints version string
bool version;
/// Print information about devices
bool device_info;
//
// Methods
//
About(CommandLine const &cmdline);
void print_usage(std::ostream &out) const;
void print_options(std::ostream &out, int indent = 0) const;
static void print_version(std::ostream &out);
};
public:
//
// Data members
//
/// Top-level execution mode
ExecutionMode execution_mode;
/// Name of math function to profile
library::OperationKind operation_kind;
/// Vector of operation name substrings
std::vector<std::string> operation_names;
/// Vector of operation name substrings
std::vector<std::string> excluded_operation_names;
//
// Detailed configuration options
//
/// Configuration
CommandLine cmdline;
Device device;
Initialization initialization;
Library library;
Verification verification;
Profiling profiling;
Report report;
About about;
public:
Options(CommandLine const &cmdline);
void print_usage(std::ostream &out) const;
void print_options(std::ostream &out) const;
static std::string indent_str(int indent);
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass

127
tools/profiler/include/cutlass/profiler/performance_report.h Normal file
View File

@@ -0,0 +1,127 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Class performing output during profiling
*/
#pragma once
#include <vector>
#include <fstream>
// CUTLASS Profiler includes
#include "options.h"
#include "enumerated_types.h"
#include "performance_result.h"
// CUTLASS Library includes
#include "cutlass/library/library.h"
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
class PerformanceReport {
private:
/// Reference to options
Options const &options_;
/// Operation kind
library::OperationKind op_kind_;
/// Operation file name containing performance report of op_kind
std::string op_file_name_;
/// Output file containing results
std::ofstream output_file_;
/// Operation file name containing junit performance report of op_kind
std::string op_junit_file_name_;
/// Output file containing junit results
std::ofstream junit_output_file_;
/// Flag indicating the performance report is valid
bool good_;
/// Vector of argument names
std::vector<std::string> argument_names_;
/// Counter uniquely identifying problem within the report
size_t problem_index_;
/// Collection of all results
PerformanceResultVector concatenated_results_;
public:
PerformanceReport(Options const &options, std::vector<std::string> const &argument_names, library::OperationKind const &op_kind);
~PerformanceReport();
bool good() const { return good_; }
void next_problem();
void append_result(PerformanceResult result);
void sort_results(PerformanceResultVector &results);
void append_results(PerformanceResultVector const &results);
public:
/// Prints the CSV header
std::ostream & print_csv_header_(std::ostream &out);
/// Prints the CSV
std::ostream & print_result_csv_(std::ostream &out, PerformanceResult const &result);
/// @defgroup jUnit Result Generation
/// Functions related to generation of the jUnit results
/// @{
std::ostream & print_junit_header_(std::ostream &out);
std::ostream & print_junit_result_(std::ostream &out, PerformanceResult const &result);
std::ostream & print_junit_footer_(std::ostream &out);
/// @}
/// Prints the result in human readable form
std::ostream & print_result_pretty_(
std::ostream &out,
PerformanceResult const &result,
bool use_shell_coloring = true);
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass

128
tools/profiler/include/cutlass/profiler/performance_result.h Normal file
View File

@@ -0,0 +1,128 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Defines a math function
*/
#pragma once
#include <vector>
#include "cutlass/cutlass.h"
// CUTLASS Profiler includes
#include "enumerated_types.h"
// CUTLASS Library includes
#include "cutlass/library/library.h"
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Performance result object
struct PerformanceResult {
/// Index of problem
size_t problem_index;
/// library::Provider
library::Provider provider;
/// Operation kind
library::OperationKind op_kind;
/// CUTLASS status result from kernels (success or failure)
// Status does information on verification
Status status;
/// Outcome of verification (worst case verification result)
Disposition disposition;
/// Outcome of verification (all verification results)
DispositionMap verification_map;
/// Operation name
std::string operation_name;
/// Stringified vector of argument values
std::vector<std::pair<std::string, std::string> > arguments;
/// Number of bytes read or written
int64_t bytes;
/// Number of DL flops performed by the math function
int64_t flops;
/// Average runtime in ms
double runtime;
//
// Members
//
/// Ctor
PerformanceResult():
problem_index(0),
op_kind(library::OperationKind::kInvalid),
provider(library::Provider::kInvalid),
disposition(Disposition::kNotRun),
status(Status::kInvalid),
bytes(0),
flops(0),
runtime(0)
{ }
/// Returns true if the runtime is valid
bool good() const {
return runtime > 0;
}
/// Math throughput in units of GFLOP/s
double gflops_per_sec() const {
return double(flops) / runtime / 1.0e6;
}
/// memory bandwidth in units of GiB/s
double gbytes_per_sec() const {
return double(bytes) / double(1 << 30) / runtime * 1000.0;
}
};
using PerformanceResultVector = std::vector<PerformanceResult>;
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass

1014
tools/profiler/include/cutlass/profiler/problem_space.h Normal file
View File

File diff suppressed because it is too large Load Diff

229
tools/profiler/include/cutlass/profiler/rank_2k_operation_profiler.h Normal file
View File

@@ -0,0 +1,229 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Defines a math function
*/
#pragma once
#include <vector>
#include <string>
#include <memory>
#include <algorithm>
#include <unordered_map>
// CUTLASS Library includes
#include "cutlass/blas3.h"
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
#include "cutlass/library/manifest.h"
// Profiler includes
#include "options.h"
#include "device_context.h"
#include "operation_profiler.h"
#include "performance_result.h"
#include "problem_space.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Abstract base class for each math function
class Rank2KOperationProfiler : public OperationProfiler {
public:
/// Problem structure obtained from problem space
struct RankKProblem {
int64_t n;
int64_t k;
int64_t lda;
int64_t ldb;
int64_t ldc;
FillMode fill_mode;
BlasMode blas_mode;
std::vector<uint8_t> alpha;
std::vector<uint8_t> beta;
int64_t split_k_slices;
int64_t batch_count;
//
// Methods
//
RankKProblem():
n(16), k(16), lda(0), ldc(0),
fill_mode(FillMode::kInvalid), blas_mode(BlasMode::kInvalid),
split_k_slices(1), batch_count(1) { }
/// Parses the problem
Status parse(
library::RankKDescription const &operation_desc,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Total number of bytes loaded
int64_t bytes(library::RankKDescription const &operation_desc) const;
/// Total number of flops computed
int64_t flops(library::RankKDescription const &operation_desc) const;
/// Initializes a performance result
void initialize_result(
PerformanceResult &result,
library::RankKDescription const &operation_desc,
ProblemSpace const &problem_space);
};
/// Workspace used
struct RankKWorkspace {
DeviceAllocation *A;
DeviceAllocation *B;
DeviceAllocation *C;
DeviceAllocation *Computed;
DeviceAllocation *Reference;
library::RankKConfiguration configuration;
library::RankKArguments arguments;
/// Buffer used for the operation's host workspace
std::vector<uint8_t> host_workspace;
/// Buffer used for the operations' device workspace
DeviceAllocation device_workspace;
//
// Methods
//
RankKWorkspace():
A(nullptr), B(nullptr), C(nullptr), Computed(nullptr), Reference(nullptr) { }
};
protected:
//
// Data members
//
/// GEMM problem obtained from problem space
RankKProblem problem_;
/// Device memory allocations
RankKWorkspace rank_k_workspace_;
public:
//
// Methods
//
/// Ctor
Rank2KOperationProfiler(Options const &options);
/// Destructor
virtual ~Rank2KOperationProfiler();
/// Prints usage statement for the math function
virtual void print_usage(std::ostream &out) const;
/// Prints examples
virtual void print_examples(std::ostream &out) const;
/// Extracts the problem dimensions
virtual Status initialize_configuration(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Initializes workspace
virtual Status initialize_workspace(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Verifies CUTLASS against references
virtual bool verify_cutlass(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Measures performance results
virtual bool profile(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
protected:
/// Initializes the performance result
void initialize_result_(
PerformanceResult &result,
Options const &options,
library::RankKDescription const &operation_desc,
ProblemSpace const &problem_space);
/// Verifies CUTLASS against references
bool verify_with_cublas_(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////

227
tools/profiler/include/cutlass/profiler/rank_k_operation_profiler.h Normal file
View File

@@ -0,0 +1,227 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Defines a math function
*/
#pragma once
#include <vector>
#include <string>
#include <memory>
#include <algorithm>
#include <unordered_map>
// CUTLASS Library includes
#include "cutlass/blas3.h"
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
#include "cutlass/library/manifest.h"
// Profiler includes
#include "options.h"
#include "device_context.h"
#include "operation_profiler.h"
#include "performance_result.h"
#include "problem_space.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Abstract base class for each math function
class RankKOperationProfiler : public OperationProfiler {
public:
/// Problem structure obtained from problem space
struct RankKProblem {
int64_t n;
int64_t k;
int64_t lda;
int64_t ldc;
FillMode fill_mode;
BlasMode blas_mode;
std::vector<uint8_t> alpha;
std::vector<uint8_t> beta;
int64_t split_k_slices;
int64_t batch_count;
//
// Methods
//
RankKProblem():
n(16), k(16), lda(0), ldc(0),
fill_mode(FillMode::kInvalid), blas_mode(BlasMode::kInvalid),
split_k_slices(1), batch_count(1) { }
/// Parses the problem
Status parse(
library::RankKDescription const &operation_desc,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Total number of bytes loaded
int64_t bytes(library::RankKDescription const &operation_desc) const;
/// Total number of flops computed
int64_t flops(library::RankKDescription const &operation_desc) const;
/// Initializes a performance result
void initialize_result(
PerformanceResult &result,
library::RankKDescription const &operation_desc,
ProblemSpace const &problem_space);
};
/// Workspace used
struct RankKWorkspace {
DeviceAllocation *A;
DeviceAllocation *C;
DeviceAllocation *Computed;
DeviceAllocation *Reference;
library::RankKConfiguration configuration;
library::RankKArguments arguments;
/// Buffer used for the operation's host workspace
std::vector<uint8_t> host_workspace;
/// Buffer used for the operations' device workspace
DeviceAllocation device_workspace;
//
// Methods
//
RankKWorkspace():
A(nullptr), C(nullptr), Computed(nullptr), Reference(nullptr) { }
};
protected:
//
// Data members
//
/// GEMM problem obtained from problem space
RankKProblem problem_;
/// Device memory allocations
RankKWorkspace rank_k_workspace_;
public:
//
// Methods
//
/// Ctor
RankKOperationProfiler(Options const &options);
/// Destructor
virtual ~RankKOperationProfiler();
/// Prints usage statement for the math function
virtual void print_usage(std::ostream &out) const;
/// Prints examples
virtual void print_examples(std::ostream &out) const;
/// Extracts the problem dimensions
virtual Status initialize_configuration(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Initializes workspace
virtual Status initialize_workspace(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Verifies CUTLASS against references
virtual bool verify_cutlass(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Measures performance results
virtual bool profile(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
protected:
/// Initializes the performance result
void initialize_result_(
PerformanceResult &result,
Options const &options,
library::RankKDescription const &operation_desc,
ProblemSpace const &problem_space);
/// Verifies CUTLASS against references
bool verify_with_cublas_(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////

173
tools/profiler/include/cutlass/profiler/reduction_operation_profiler.h Normal file
View File

@@ -0,0 +1,173 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Defines profiling functionality for reduction operation
*/
#pragma once
#include <vector>
#include <string>
#include <memory>
#include <algorithm>
#include <unordered_map>
// CUTLASS Library includes
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
#include "cutlass/library/manifest.h"
// Profiler includes
#include "options.h"
#include "device_context.h"
#include "operation_profiler.h"
#include "performance_result.h"
#include "problem_space.h"
#if CUTLASS_ENABLE_CUDNN
#include "cudnn_helpers.h"
#endif //#if CUTLASS_ENABLE_CUDNN
#include "debug.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Abstract base class for each math function
class ReductionOperationProfiler : public OperationProfiler {
public:
/// Workspace used
struct ReductionWorkspace {
/// Conv device allocations
DeviceAllocation *Workspace;
DeviceAllocation *Source;
DeviceAllocation *Destination;
DeviceAllocation *Reference;
/// Library configuration and arguments
library::ReductionConfiguration configuration;
library::ReductionArguments arguments;
/// Buffer used for the cutlass operations' host workspace
std::vector<uint8_t> host_workspace;
/// Buffer used for the cutlass operations' device workspace
DeviceAllocation device_workspace;
//
// Methods
//
ReductionWorkspace():
Workspace(nullptr), Source(nullptr), Destination(nullptr), Reference(nullptr) { }
};
protected:
//
// Data members
//
/// Reduction problem obtained from problem space
MatrixCoord problem_;
/// Device memory allocations
ReductionWorkspace conv_workspace_;
public:
//
// Methods
//
/// Ctor
ReductionOperationProfiler(Options const &options);
/// Destructor
virtual ~ReductionOperationProfiler();
/// Prints usage statement for the math function
virtual void print_usage(std::ostream &out) const;
/// Prints examples
virtual void print_examples(std::ostream &out) const;
/// Extracts the problem dimensions
virtual Status initialize_configuration(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Initializes workspace
virtual Status initialize_workspace(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Verifies CUTLASS against references
virtual bool verify_cutlass(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Measures performance results
virtual bool profile(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////

214
tools/profiler/include/cutlass/profiler/sparse_gemm_operation_profiler.h Normal file
View File

@@ -0,0 +1,214 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief
*/
#pragma once
#include <vector>
#include <string>
#include <memory>
#include <algorithm>
#include <unordered_map>
// CUTLASS Library includes
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
#include "cutlass/library/manifest.h"
// Profiler includes
#include "options.h"
#include "device_context.h"
#include "operation_profiler.h"
#include "performance_result.h"
#include "problem_space.h"
#include "gemm_operation_profiler.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Abstract base class for each math function
class SparseGemmOperationProfiler : public OperationProfiler {
public:
/// Problem structure obtained from problem space
struct SparseGemmProblem {
int64_t m;
int64_t n;
int64_t k;
int64_t lda;
int64_t ldb;
int64_t ldc;
int64_t lde;
std::vector<uint8_t> alpha;
std::vector<uint8_t> beta;
int64_t split_k_slices;
int64_t batch_count;
static int const sparse = 2;
// every 128b ElementA uses one elementE
int elements_per_128b;
//
// Methods
//
SparseGemmProblem():
m(16), n(16), k(16), lda(0), ldb(0), ldc(0), lde(0), split_k_slices(1), batch_count(1) { }
/// Parses the problem
Status parse(
library::SparseGemmDescription const &operation_desc,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Initializes a performance result
void initialize_result(
PerformanceResult &result,
library::SparseGemmDescription const &operation_desc,
ProblemSpace const &problem_space);
};
/// Workspace used
struct SparseGemmWorkspace {
DeviceAllocation *A;
DeviceAllocation *B;
DeviceAllocation *C;
DeviceAllocation *E;
DeviceAllocation *Computed;
DeviceAllocation *Reference;
library::SparseGemmConfiguration configuration;
library::SparseGemmArguments arguments;
/// Buffer used for the operation's host workspace
std::vector<uint8_t> host_workspace;
/// Buffer used for the operations' device workspace
DeviceAllocation device_workspace;
//
// Methods
//
SparseGemmWorkspace():
A(nullptr), B(nullptr), C(nullptr), E(nullptr), Computed(nullptr), Reference(nullptr) { }
};
protected:
//
// Data members
//
// GEMM problem
SparseGemmProblem problem_;
/// Device memory allocations
SparseGemmWorkspace gemm_workspace_;
public:
//
// Methods
//
/// Ctor
SparseGemmOperationProfiler(Options const &options);
/// Destructor
virtual ~SparseGemmOperationProfiler();
/// Prints usage statement for the math function
virtual void print_usage(std::ostream &out) const;
/// Prints examples
virtual void print_examples(std::ostream &out) const;
/// Extracts the problem dimensions
virtual Status initialize_configuration(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Initializes workspace
virtual Status initialize_workspace(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Verifies CUTLASS against references
virtual bool verify_cutlass(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Measures performance results
virtual bool profile(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
protected:
/// Initializes the performance result
void initialize_result_(
PerformanceResult &result,
Options const &options,
library::SparseGemmDescription const &operation_desc,
ProblemSpace const &problem_space);
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////

230
tools/profiler/include/cutlass/profiler/symm_operation_profiler.h Normal file
View File

@@ -0,0 +1,230 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Defines a math function
*/
#pragma once
#include <vector>
#include <string>
#include <memory>
#include <algorithm>
#include <unordered_map>
// CUTLASS Library includes
#include "cutlass/blas3.h"
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
#include "cutlass/library/manifest.h"
// Profiler includes
#include "options.h"
#include "device_context.h"
#include "operation_profiler.h"
#include "performance_result.h"
#include "problem_space.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Abstract base class for each math function
class SymmOperationProfiler : public OperationProfiler {
public:
/// Problem structure obtained from problem space
struct SymmProblem {
int64_t m;
int64_t n;
int64_t lda;
int64_t ldb;
int64_t ldc;
SideMode side_mode;
FillMode fill_mode;
BlasMode blas_mode;
std::vector<uint8_t> alpha;
std::vector<uint8_t> beta;
int64_t split_k_slices;
int64_t batch_count;
//
// Methods
//
SymmProblem():
m(16), n(16), lda(0), ldb(0), ldc(0),
side_mode(SideMode::kInvalid), fill_mode(FillMode::kInvalid), blas_mode(BlasMode::kInvalid),
split_k_slices(1), batch_count(1) { }
/// Parses the problem
Status parse(
library::SymmDescription const &operation_desc,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Total number of bytes loaded
int64_t bytes(library::SymmDescription const &operation_desc) const;
/// Total number of flops computed
int64_t flops(library::SymmDescription const &operation_desc) const;
/// Initializes a performance result
void initialize_result(
PerformanceResult &result,
library::SymmDescription const &operation_desc,
ProblemSpace const &problem_space);
};
/// Workspace used
struct SymmWorkspace {
DeviceAllocation *A;
DeviceAllocation *B;
DeviceAllocation *C;
DeviceAllocation *Computed;
DeviceAllocation *Reference;
library::SymmConfiguration configuration;
library::SymmArguments arguments;
/// Buffer used for the operation's host workspace
std::vector<uint8_t> host_workspace;
/// Buffer used for the operations' device workspace
DeviceAllocation device_workspace;
//
// Methods
//
SymmWorkspace():
A(nullptr), B(nullptr), C(nullptr), Computed(nullptr), Reference(nullptr) { }
};
protected:
//
// Data members
//
/// GEMM problem obtained from problem space
SymmProblem problem_;
/// Device memory allocations
SymmWorkspace symm_workspace_;
public:
//
// Methods
//
/// Ctor
SymmOperationProfiler(Options const &options);
/// Destructor
virtual ~SymmOperationProfiler();
/// Prints usage statement for the math function
virtual void print_usage(std::ostream &out) const;
/// Prints examples
virtual void print_examples(std::ostream &out) const;
/// Extracts the problem dimensions
virtual Status initialize_configuration(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Initializes workspace
virtual Status initialize_workspace(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Verifies CUTLASS against references
virtual bool verify_cutlass(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Measures performance results
virtual bool profile(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
protected:
/// Initializes the performance result
void initialize_result_(
PerformanceResult &result,
Options const &options,
library::SymmDescription const &operation_desc,
ProblemSpace const &problem_space);
/// Verifies CUTLASS against references
bool verify_with_cublas_(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////

222
tools/profiler/include/cutlass/profiler/trmm_operation_profiler.h Normal file
View File

@@ -0,0 +1,222 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Defines a math function
*/
#pragma once
#include <vector>
#include <string>
#include <memory>
#include <algorithm>
#include <unordered_map>
// CUTLASS Library includes
#include "cutlass/blas3.h"
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
#include "cutlass/library/manifest.h"
// Profiler includes
#include "options.h"
#include "device_context.h"
#include "operation_profiler.h"
#include "performance_result.h"
#include "problem_space.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Abstract base class for each math function
class TrmmOperationProfiler : public OperationProfiler {
public:
/// Problem structure obtained from problem space
struct TrmmProblem {
int64_t m;
int64_t n;
int64_t lda;
int64_t ldb;
int64_t ldd;
SideMode side_mode;
FillMode fill_mode;
DiagType diag_type;
std::vector<uint8_t> alpha;
std::vector<uint8_t> beta;
int64_t split_k_slices;
int64_t batch_count;
//
// Methods
//
TrmmProblem():
m(16), n(16), lda(0), ldb(0), ldd(0), split_k_slices(1), batch_count(1) { }
/// Parses the problem
Status parse(
library::TrmmDescription const &operation_desc,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Initializes a performance result
void initialize_result(
PerformanceResult &result,
library::TrmmDescription const &operation_desc,
ProblemSpace const &problem_space);
};
/// Workspace used
struct TrmmWorkspace {
DeviceAllocation *A;
DeviceAllocation *B;
DeviceAllocation *D;
DeviceAllocation *Computed;
DeviceAllocation *Reference;
library::TrmmConfiguration configuration;
library::TrmmArguments arguments;
/// Buffer used for the operation's host workspace
std::vector<uint8_t> host_workspace;
/// Buffer used for the operations' device workspace
DeviceAllocation device_workspace;
//
// Methods
//
TrmmWorkspace():
A(nullptr), B(nullptr), D(nullptr), Computed(nullptr), Reference(nullptr) { }
};
protected:
//
// Data members
//
/// GEMM problem obtained from problem space
TrmmProblem problem_;
/// Device memory allocations
TrmmWorkspace trmm_workspace_;
public:
//
// Methods
//
/// Ctor
TrmmOperationProfiler(Options const &options);
/// Destructor
virtual ~TrmmOperationProfiler();
/// Prints usage statement for the math function
virtual void print_usage(std::ostream &out) const;
/// Prints examples
virtual void print_examples(std::ostream &out) const;
/// Extracts the problem dimensions
virtual Status initialize_configuration(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Initializes workspace
virtual Status initialize_workspace(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Verifies CUTLASS against references
virtual bool verify_cutlass(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Measures performance results
virtual bool profile(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
protected:
/// Initializes the performance result
void initialize_result_(
PerformanceResult &result,
Options const &options,
library::TrmmDescription const &operation_desc,
ProblemSpace const &problem_space);
/// Verifies CUTLASS against references
bool verify_with_cublas_(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////