leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 30 Packages Projects Releases Wiki Activity

3.6.0 update (#2005)

Browse Source 
* 3.6.0 update

* doc and swap stuff

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
Yujia Zhai
2024-12-25 01:34:40 -05:00
committed by GitHub
co-authored by yuzhai Haicheng Wu
parent e1cd8c7866
commit 3d261a5974
258 changed files with 10863 additions and 3883 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/library/include/cutlass/library/handle.h
+2
View File
@@ -72,6 +72,8 @@ private:
/// Pointer to the most recently executed operation
Operation const *last_operation_;
int device_idx_;
public:
/// Constructor
tools/library/include/cutlass/library/library.h
+6 -2
View File
@@ -103,7 +103,6 @@ public:
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const = 0;
// Originally designed for metadata, but should be useful for FP8/6/4 too.
virtual Status initialize_with_profiler_workspace(
void const *configuration,
void *host_workspace,
@@ -118,7 +117,8 @@ public:
void const *arguments,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const = 0;
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const = 0;
};
@@ -272,6 +272,8 @@ struct GemmUniversalConfiguration {
int64_t ldb{0};
int64_t ldc{0};
int64_t ldd{0};
int device_count{1};
};
struct GemmUniversalArguments {
@@ -303,6 +305,8 @@ struct GemmUniversalArguments {
int sm_count{0};
library::RasterOrder raster_order{};
int swizzle_size{1};
int device_index{0};
};
/////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/conv2d_operation.h
+12 -2
View File
@@ -326,7 +326,12 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const {
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
OperatorArguments args;
@@ -578,7 +583,12 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const {
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
OperatorArguments args;
tools/library/src/conv3d_operation.h
+6 -1
View File
@@ -317,7 +317,12 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const {
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
OperatorArguments args;
tools/library/src/conv_operation_3x.hpp
+136 -22
View File
@@ -236,12 +236,14 @@ public:
typename Operator::Arguments out_args{};
status = update_operator_arguments_from_configuration_2d_or_3d(out_args, configuration);
if (status != Status::kSuccess) {
CUTLASS_TRACE_HOST("*** can_implement: update_operator_arguments_from_configuration_2d_or_3d failed");
return status;
}
auto* in_args_ptr = reinterpret_cast<ConvArguments const*>(arguments);
status = update_operator_arguments_from_arguments(out_args, *in_args_ptr);
if (status != Status::kSuccess) {
CUTLASS_TRACE_HOST("*** can_implement: update_operator_arguments_from_arguments failed");
return status;
}
@@ -332,7 +334,8 @@ public:
void const* arguments,
void* host_workspace,
void* device_workspace = nullptr,
cudaStream_t stream = nullptr) const override
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const override
{
auto status = Status::kInvalid;
@@ -358,7 +361,7 @@ public:
}
auto* op = reinterpret_cast<Operator*>(host_workspace);
return op->run(out_args, device_workspace, stream);
return op->run(out_args, device_workspace, stream, nullptr, launch_with_pdl);
}
private:
@@ -482,6 +485,11 @@ private:
typename Operator::Arguments& out_args,
Conv2dConfiguration const& config)
{
#if defined(CUTLASS_DEBUG_TRACE_LEVEL) && (CUTLASS_DEBUG_TRACE_LEVEL > 1)
CUTLASS_TRACE_HOST("ConvOperator3x::"
"update_operator_arguments_from_configuration"
"(Conv2dConfiguration)\n");
#endif
using detail::vector_to_array_strides;
constexpr int num_spatial_dims = Operator::NumSpatialDimensions;
@@ -595,6 +603,7 @@ private:
const TensorStride stride_A = vector_to_array_strides(config.stride_a, the_stride_size);
const TensorStride stride_B = vector_to_array_strides(config.stride_b, the_stride_size);
const TensorStride stride_C = vector_to_array_strides(config.stride_c, the_stride_size);
// cutlass::library::Conv2dConfiguration has no member stride_d.
// The code below imitates the testbed,
@@ -605,12 +614,57 @@ private:
CUTLASS_TRACE_HOST("CUTLASS 3 kernels currently only support groups = 1.");
return Status::kInvalid;
}
// ConvProblemShape is how CUTLASS 3 kernels represent
// convolution problems. ConvProblemShape's constructors take
// shape_act, stride_act, shape_flt, and stride_flt, and set
// shape_A, stride_A, shape_B, stride_B, shape_C, and stride_C
// according to Fprop / Dgrad / Wgrad.
//
// This means that stride_act isn't always config.stride_A,
// depending on Fprop / Dgrad / Wgrad. The code here "undoes"
// the logic in Conv2dWorkspace::set_stride_vector so that we
// can recover the strides of the activation and filter tensors.
// It doesn't need to worry about the so-called "output" tensor
// (which might not be C), as ConvProblemShape's constructor
// figures out its shapes and strides.
using TensorExtent = typename problem_shape_type::TensorExtent;
TensorExtent shape_act{N, H, W, C};
auto stride_act = [&] () {
// Some compilers consider conv_op (defined above), as
// captured by this lambda, as "not a constant expression."
constexpr auto conv_kind = Operator::DispatchPolicy::ConvOp;
if constexpr (conv_kind == cutlass::conv::Operator::kFprop) {
return stride_A;
}
else if constexpr (conv_kind == cutlass::conv::Operator::kDgrad) {
return stride_C;
}
else { // conv_kind == cutlass::conv::Operator::kWgrad
return stride_B;
}
} ();
TensorExtent shape_flt{K, R, S, C};
auto stride_flt = [&] () {
// Some compilers consider conv_op (defined above), as
// captured by this lambda, as "not a constant expression."
constexpr auto conv_kind = Operator::DispatchPolicy::ConvOp;
if constexpr (conv_kind == cutlass::conv::Operator::kFprop) {
return stride_B;
}
else if constexpr (conv_kind == cutlass::conv::Operator::kDgrad) {
return stride_B;
}
else { // conv_kind == cutlass::conv::Operator::kWgrad
return stride_C;
}
} ();
problem_shape_type problem_shape(
/* mode = */ mode,
/* shape_act = */ {N, H, W, C},
/* stride_act = */ stride_A,
/* shape_flt = */ {K, R, S, C},
/* stride_flt = */ stride_B,
/* shape_act = */ shape_act,
/* stride_act = */ stride_act,
/* shape_flt = */ shape_flt,
/* stride_flt = */ stride_flt,
/* lower_padding = */ {pad_h, pad_w},
/* upper_padding = */ {pad_h, pad_w},
/* traversal_stride = */ {traversal_stride_h, traversal_stride_w},
@@ -620,9 +674,11 @@ private:
// ConvProblemShape's constructor sets its shape_C member.
#if defined(CUTLASS_DEBUG_TRACE_LEVEL) && (CUTLASS_DEBUG_TRACE_LEVEL > 1)
std::cerr << " problem_shape:\n"
<< " shape_C: " << problem_shape.shape_C << "\n";
std::cerr << " stride_C: " << problem_shape.stride_C << "\n";
printf("\n problem_shape.shape_C: ");
print(problem_shape.shape_C);
printf("\n problem_shape.stride_C: ");
print(problem_shape.stride_C);
printf("\n");
#endif
// Initialization of C's and D's strides follows the CUTLASS 3
// convolutions testbed (test/unit/conv/device_3x/testbed_conv.hpp).
@@ -670,6 +726,11 @@ private:
typename Operator::Arguments& out_args,
Conv3dConfiguration const& config)
{
#if defined(CUTLASS_DEBUG_TRACE_LEVEL) && (CUTLASS_DEBUG_TRACE_LEVEL > 1)
CUTLASS_TRACE_HOST("ConvOperator3x::"
"update_operator_arguments_from_configuration"
"(Conv3dConfiguration)\n");
#endif
using detail::coord_to_array_strides;
constexpr int num_spatial_dims = Operator::NumSpatialDimensions;
@@ -762,6 +823,10 @@ private:
print_stride(input_stride_b, "input_stride_b");
print_stride(input_stride_c, "input_stride_c");
#endif
// Conv3dConfiguration stores the strides as Coord (with
// compile-time size), so there's no need to check sizes here
// (unlike Conv2dConfiguration, which stores strides as
// std::vector).
constexpr cutlass::conv::Operator conv_op = Operator::DispatchPolicy::ConvOp;
using problem_shape_type =
@@ -771,18 +836,68 @@ private:
const TensorStride stride_A = coord_to_array_strides(input_stride_a);
const TensorStride stride_B = coord_to_array_strides(input_stride_b);
const TensorStride stride_C = coord_to_array_strides(input_stride_c);
const int num_groups = config.problem_size.groups;
if (num_groups != 1) {
CUTLASS_TRACE_HOST("CUTLASS 3 kernels currently only support groups = 1.");
return Status::kInvalid;
}
// ConvProblemShape is how CUTLASS 3 kernels represent
// convolution problems. ConvProblemShape's constructors take
// shape_act, stride_act, shape_flt, and stride_flt, and set
// shape_A, stride_A, shape_B, stride_B, shape_C, and stride_C
// according to Fprop / Dgrad / Wgrad.
//
// Conv3dConfiguration differs a bit from Conv2dConfiguration,
// but the idea is the same: the "input_stride_a" from config
// depends on conv_kind (Fprop, Dgrad, or Wgrad), so stride_act
// isn't always input_stride_a. Analogously, stride_flt isn't
// always input_stride_b. The code here "undoes" the logic in
// config.layout_a(conv_kind) and config.layout_b(conv_kind)
// (analogous to Conv2dWorkspace::set_stride_vector) so that we
// can recover the strides of the activation and filter tensors.
// It doesn't need to worry about the so-called "output" tensor
// (which might not be C), as ConvProblemShape's constructor
// figures out its shapes and strides.
using TensorExtent = typename problem_shape_type::TensorExtent;
TensorExtent shape_act{N, D, H, W, C};
auto stride_act = [&] () {
// Some compilers consider conv_op (defined above), as
// captured by this lambda, as "not a constant expression."
constexpr auto conv_kind = Operator::DispatchPolicy::ConvOp;
if constexpr (conv_kind == cutlass::conv::Operator::kFprop) {
return stride_A;
}
else if constexpr (conv_kind == cutlass::conv::Operator::kDgrad) {
return stride_C;
}
else { // conv_kind == cutlass::conv::Operator::kWgrad
return stride_B;
}
} ();
TensorExtent shape_flt{K, T, R, S, C};
auto stride_flt = [&] () {
// Some compilers consider conv_op (defined above), as
// captured by this lambda, as "not a constant expression."
constexpr auto conv_kind = Operator::DispatchPolicy::ConvOp;
if constexpr (conv_kind == cutlass::conv::Operator::kFprop) {
return stride_B;
}
else if constexpr (conv_kind == cutlass::conv::Operator::kDgrad) {
return stride_B;
}
else { // conv_kind == cutlass::conv::Operator::kWgrad
return stride_C;
}
} ();
problem_shape_type problem_shape(
/* mode = */ mode,
/* shape_act = */ {N, D, H, W, C},
/* stride_act = */ stride_A,
/* shape_flt = */ {K, T, R, S, C},
/* stride_flt = */ stride_B,
/* shape_act = */ shape_act,
/* stride_act = */ stride_act,
/* shape_flt = */ shape_flt,
/* stride_flt = */ stride_flt,
/* lower_padding = */ {pad_d, pad_h, pad_w},
/* upper_padding = */ {pad_d, pad_h, pad_w},
/* traversal_stride = */ {traversal_stride_d, traversal_stride_h, traversal_stride_w},
@@ -792,15 +907,15 @@ private:
// ConvProblemShape's constructor sets its shape_C member.
#if defined(CUTLASS_DEBUG_TRACE_LEVEL) && (CUTLASS_DEBUG_TRACE_LEVEL > 1)
std::cerr << " problem_shape:\n"
<< " shape_C: " << problem_shape.shape_C << "\n";
std::cerr << " stride_C: " << problem_shape.stride_C << "\n";
printf("\n problem_shape.shape_C: ");
print(problem_shape.shape_C);
printf("\n problem_shape.stride_C: ");
print(problem_shape.stride_C);
printf("\n");
#endif
// Initialization of C's and D's strides follows the CUTLASS 3
// convolutions testbed (test/unit/conv/device_3x/testbed_conv.hpp).
{
#if defined(CUTLASS_DEBUG_TRACE_LEVEL) && (CUTLASS_DEBUG_TRACE_LEVEL > 1)
std::cerr << " Compute stride_C and stride_D\n";
#endif
using StrideC = typename Operator::ConvKernel::StrideC;
using StrideD = typename Operator::ConvKernel::StrideD;
auto stride_C = StrideC{};
@@ -845,9 +960,8 @@ private:
ConvArguments const& in_args) const
{
#if defined(CUTLASS_DEBUG_TRACE_LEVEL) && (CUTLASS_DEBUG_TRACE_LEVEL > 1)
std::cerr << "ConvOperation3x::update_operator_arguments_from_arguments\n";
CUTLASS_TRACE_HOST("ConvOperation3x::update_operator_arguments_from_arguments\n");
#endif
auto status = UpdateFusionArgs<decltype(out_args.epilogue.thread)>::update_(
out_args.epilogue.thread, in_args);
if (status != Status::kSuccess) {
tools/library/src/gemm_operation.h
+36 -6
View File
@@ -296,7 +296,12 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const {
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
OperatorArguments args;
@@ -500,7 +505,12 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const {
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
OperatorArguments args;
@@ -721,7 +731,12 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const {
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
OperatorArguments args;
@@ -930,7 +945,12 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const {
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
OperatorArguments args;
@@ -1133,7 +1153,12 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const {
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
OperatorArguments args;
@@ -1337,7 +1362,12 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const {
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
OperatorArguments args;
tools/library/src/gemm_operation_3x.hpp
+4 -2
View File
@@ -35,6 +35,7 @@
#pragma once
#include "cutlass/cutlass.h"
#include "cutlass/detail/collective.hpp"
#include "cutlass/library/library.h"
#include "library_internal.h"
#include "cutlass/gemm/dispatch_policy.hpp"
@@ -331,7 +332,8 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const override {
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const override {
OperatorArguments args;
Status status = update_arguments_(args, static_cast<GemmUniversalArguments const *>(arguments_ptr));
@@ -341,7 +343,7 @@ public:
Operator *op = static_cast<Operator *>(host_workspace);
// We need to call initialize() since we have to rebuild TMA desc for every new set of args
status = op->run(args, device_workspace, stream);
status = op->run(args, device_workspace, stream, nullptr, launch_with_pdl);
return status;
}
};
tools/library/src/handle.cu
+25 -6
View File
@@ -57,14 +57,12 @@ Handle::Handle(
scalar_pointer_mode_(ScalarPointerMode::kHost),
last_operation_(nullptr) {
int device_idx = -1;
cudaError_t error = cudaGetDevice(&device_idx);
cudaError_t error = cudaGetDevice(&device_idx_);
if (error != cudaSuccess) {
throw std::runtime_error("cudaGetDevice() failed");
}
error = cudaGetDeviceProperties(&device_, device_idx);
error = cudaGetDeviceProperties(&device_, device_idx_);
if (error != cudaSuccess) {
throw std::runtime_error("cudaGetDeviceProperties() failed");
}
@@ -78,8 +76,14 @@ Handle::Handle(
Handle::~Handle() {
if (workspace_) {
if (workspace_) {
cudaFree(workspace_);
int device_before;
cudaGetDevice(&device_before);
if (device_before != device_idx_) {
cudaSetDevice(device_idx_);
}
cudaFree(workspace_);
if (device_before != device_idx_) {
cudaSetDevice(device_before);
}
workspace_ = nullptr;
@@ -89,6 +93,10 @@ Handle::~Handle() {
/// Move constructor
Handle::Handle(Handle && handle) {
cudaError_t error = cudaGetDevice(&device_idx_);
if (error != cudaSuccess) {
throw std::runtime_error("cudaGetDevice() failed");
}
device_ = handle.device_;
workspace_size_ = handle.workspace_size_;
workspace_ = handle.workspace_;
@@ -112,6 +120,8 @@ Handle & Handle::operator=(Handle && handle) {
handle.workspace_ = nullptr;
handle.workspace_size_ = 0;
device_idx_ = handle.device_idx_;
return *this;
}
@@ -151,6 +161,12 @@ void *Handle::get_workspace() const {
/// Sets the size of device workspace, invalidating previous calls to get_device_workspace()
void Handle::set_workspace_size(size_t bytes) {
int device_before;
cudaGetDevice(&device_before);
if (device_before != device_idx_) {
cudaSetDevice(device_idx_);
}
if (bytes != workspace_size_) {
if (workspace_) {
@@ -177,6 +193,9 @@ void Handle::set_workspace_size(size_t bytes) {
throw std::runtime_error("Failed to clear workspace");
}
}
if (device_before != device_idx_) {
cudaSetDevice(device_before);
}
}
/// Gets the scalar pointer mode
tools/library/src/library_internal.h
+8
View File
@@ -72,6 +72,10 @@ template <> struct NumericTypeMap<cutlass::uint1b_t> {
static NumericTypeID const kId = NumericTypeID::kB1;
};
template <> struct NumericTypeMap<cutlass::int2b_t> {
static NumericTypeID const kId = NumericTypeID::kS2;
};
template <> struct NumericTypeMap<cutlass::int4b_t> {
static NumericTypeID const kId = NumericTypeID::kS4;
};
@@ -92,6 +96,10 @@ template <> struct NumericTypeMap<int64_t> {
static NumericTypeID const kId = NumericTypeID::kS64;
};
template <> struct NumericTypeMap<cutlass::uint2b_t> {
static NumericTypeID const kId = NumericTypeID::kU2;
};
template <> struct NumericTypeMap<cutlass::uint4b_t> {
static NumericTypeID const kId = NumericTypeID::kU4;
};
tools/library/src/rank_2k_operation.h
+6 -1
View File
@@ -314,7 +314,12 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const {
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
OperatorArguments args;
tools/library/src/rank_k_operation.h
+6 -1
View File
@@ -310,7 +310,12 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const {
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
OperatorArguments args;
tools/library/src/reduction/reduction_operation.h
+6 -1
View File
@@ -231,7 +231,12 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const {
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
OperatorArguments args;
tools/library/src/reference/conv_reference_operation.h
+6 -1
View File
@@ -432,7 +432,12 @@ public:
void const *arguments,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const {
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
ConvArguments const &args = *static_cast<ConvArguments const *>(arguments);
tools/library/src/reference/gemm_reference_operation.h
+6 -1
View File
@@ -192,7 +192,12 @@ public:
void const *arguments,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const {
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
GemmUniversalConfiguration const &config = *static_cast<GemmUniversalConfiguration const *>(host_workspace);
GemmUniversalArguments const &args = *static_cast<GemmUniversalArguments const *>(arguments);
tools/library/src/reference/gemm_s8_s8_s32.cu
+1 -1
View File
@@ -77,7 +77,7 @@ void initialize_gemm_reference_operations_s8_s8_s32(Manifest &manifest) {
int8_t, // ElementA
int8_t, // ElementB
int32_t, // ElementC
int32_t, // ElementScalar / ElementCompute
float, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
int32_t // ElementD
>(manifest);
tools/library/src/sparse_gemm_operation_3x.hpp
+26 -25
View File
@@ -35,6 +35,7 @@
#pragma once
#include "cutlass/cutlass.h"
#include "cutlass/detail/collective.hpp"
#include "cutlass/library/library.h"
#include "cutlass/transform/kernel/sparse_gemm_compressor.hpp" // StructuredSparseCompressor
#include "cutlass/transform/device/transform_universal_adapter.hpp" // TransformUniversalAdapter
@@ -56,7 +57,7 @@ namespace cutlass::library {
///////////////////////////////////////////////////////////////////////////////////////////////////
// Limitation & Assumptions:
// Limitation & Assumptions:
// 1. The tensor must be densely packed. That is, lda is k if the tensor is k-major,
// and lda is m if the tensor is m-major.
// 2. Circular buffer for tensorA and tensorE may have a less count compared to tensorB and others.
@@ -169,7 +170,6 @@ protected:
return status;
}
// TODO: type erase Arguments structure in 3.0 GEMM
operator_args.problem_shape = cute::make_shape(
arguments->problem_size.m(),
arguments->problem_size.n(),
@@ -302,13 +302,15 @@ public:
}
Status initialize_with_profiler_workspace(
void const *configuration,
void *host_workspace,
void *device_workspace,
void const *configuration,
void *host_workspace,
void *device_workspace,
uint8_t **profiler_workspaces,
int problem_count_from_profiler,
cudaStream_t stream = nullptr) {
iter_idx.resize(static_cast<GemmUniversalConfiguration const*>(configuration)->device_count, 0);
// Set problem_count.
problem_count = problem_count_from_profiler;
@@ -319,13 +321,10 @@ public:
// * Construct Op
Operator *op = new (host_op_workspace_ptr) Operator;
// * Device Full Ptr
device_full_ptr = reinterpret_cast<uint8_t*>(device_workspace);
// * Device Ptr (1st iteration)
// Device workspace : | iter1 | iter2 | iter3 | .. | iterx |
// iteri : op_workspace | tensor_ac | tensor_e
auto* device_ptr_iter1 = device_full_ptr;
auto* device_ptr_iter1 = static_cast<uint8_t*>(device_workspace);
auto* device_op_workspace_ptr_iter1 = device_ptr_iter1;
auto* device_compressor_workspace_ptr_iter1 = device_op_workspace_ptr_iter1 + device_op_workspace_size;
auto* device_a_compressed_ptr_iter1 = device_compressor_workspace_ptr_iter1 + device_compress_workspace_size;
@@ -335,15 +334,15 @@ public:
auto* device_a_raw_ptr = profiler_workspaces[0];
// * Random fill 50% of TensorA w/ zero following the structured sparse requirement
cudaMemcpy(host_a_raw_ptr, device_a_raw_ptr, tensor_a_size, cudaMemcpyDeviceToHost);
CUDA_CHECK(cudaMemcpyAsync(host_a_raw_ptr, device_a_raw_ptr, tensor_a_size, cudaMemcpyDeviceToHost, stream));
compressor_utility.structure_sparse_zero_mask_fill(host_a_raw_ptr, 2000);
cudaMemcpy(device_a_raw_ptr, host_a_raw_ptr, tensor_a_size, cudaMemcpyHostToDevice);
CUDA_CHECK(cudaMemcpyAsync(device_a_raw_ptr, host_a_raw_ptr, tensor_a_size, cudaMemcpyHostToDevice, stream));
CUDA_CHECK(cudaGetLastError());
// * Compress DTensorA and get DTensorAC & DTensorE
cutlass::KernelHardwareInfo hw_info;
hw_info.device_id = 0;
CUDA_CHECK(cudaGetDevice(&hw_info.device_id));
hw_info.sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id);
typename Compressor::Arguments arguments{
{compressor_utility.M, 0, compressor_utility.K, compressor_utility.L},
@@ -372,23 +371,23 @@ public:
return status;
}
CUDA_CHECK(cudaStreamSynchronize(stream));
// * Copy Iter1's DTensorAC DTensorE to each iteration's DTensorAC DTensorE
for (int iter_i = 1; iter_i < problem_count; iter_i++) {
// * Device AC E Ptr per iteration
// Device workspace : | iter1 | iter2 | iter3 | .. | iterx |
// iteri : op_workspace | tensor_ac | tensor_e
auto* device_ptr_iteri = device_full_ptr + device_per_iter_workspace_size * iter_i;
auto* device_ptr_iteri = static_cast<uint8_t*>(device_workspace) + device_per_iter_workspace_size * iter_i;
auto* device_op_workspace_ptr = device_ptr_iteri;
auto* device_compressor_workspace_ptr = device_op_workspace_ptr + device_op_workspace_size;
auto* device_a_compressed_ptr = device_compressor_workspace_ptr + device_compress_workspace_size;
auto* device_e_ptr = device_a_compressed_ptr + tensor_ac_size;
cudaMemcpy(device_a_compressed_ptr, device_a_compressed_ptr_iter1, tensor_ac_size, cudaMemcpyDeviceToDevice);
cudaMemcpy(device_e_ptr, device_e_ptr_iter1, tensor_e_size, cudaMemcpyDeviceToDevice);
CUDA_CHECK(cudaMemcpyAsync(device_a_compressed_ptr, device_a_compressed_ptr_iter1, tensor_ac_size, cudaMemcpyDeviceToDevice, stream));
CUDA_CHECK(cudaMemcpyAsync(device_e_ptr, device_e_ptr_iter1, tensor_e_size, cudaMemcpyDeviceToDevice, stream));
}
CUDA_CHECK(cudaStreamSynchronize(stream));
CUDA_CHECK(cudaGetLastError());
return Status::kSuccess;
@@ -398,17 +397,21 @@ public:
Status run(
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const override {
void *device_workspace,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const override {
OperatorArguments operator_args;
auto* device_ptr_iteri = device_full_ptr + device_per_iter_workspace_size * iter_idx;
const auto device_index = static_cast<GemmUniversalArguments const *>(arguments_ptr)->device_index;
auto* device_ptr_iteri = static_cast<uint8_t*>(device_workspace) + device_per_iter_workspace_size * iter_idx[device_index];
auto* device_op_workspace_ptr = device_ptr_iteri;
auto* device_compressor_workspace_ptr = device_op_workspace_ptr + device_op_workspace_size;
auto* device_a_compressed_ptr = device_compressor_workspace_ptr + device_compress_workspace_size;
auto* device_e_ptr = device_a_compressed_ptr + tensor_ac_size;
iter_idx = (iter_idx + 1) % problem_count;
iter_idx[device_index] = (iter_idx[device_index] + 1) % problem_count;
Status status = update_arguments_(operator_args, static_cast<GemmUniversalArguments const *>(arguments_ptr), compressor_utility, device_a_compressed_ptr, device_e_ptr );
@@ -418,7 +421,7 @@ public:
Operator *op = static_cast<Operator *>(host_workspace);
// We need to call initialize() since we have to rebuild TMA desc for every new set of args
status = op->run(operator_args, device_op_workspace_ptr, stream);
status = op->run(operator_args, device_op_workspace_ptr, stream, nullptr, launch_with_pdl);
return status;
}
@@ -426,9 +429,7 @@ private:
// Variables that must change in the const functions.
mutable CompressorUtility compressor_utility;
mutable int problem_count = 1;
mutable int iter_idx = 0;
uint8_t* device_full_ptr = nullptr;
mutable std::vector<int> iter_idx;
mutable uint64_t tensor_ac_size = 0;
mutable uint64_t tensor_e_size = 0;
tools/library/src/symm_operation.h
+6 -1
View File
@@ -312,7 +312,12 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const {
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
OperatorArguments args;
tools/library/src/trmm_operation.h
+6 -1
View File
@@ -304,7 +304,12 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const {
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
OperatorArguments args;
tools/profiler/CMakeLists.txt
+1 -1
View File
@@ -100,7 +100,7 @@ install(
if (CUDA_VERSION VERSION_GREATER_EQUAL 12.3 AND CUDA_VERSION VERSION_LESS 12.4 AND (90a IN_LIST CUTLASS_NVCC_ARCHS_ENABLED OR (90 IN_LIST CUTLASS_NVCC_ARCHS_ENABLED)))
set(CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_GEMM --operation=Gemm --providers=cutlass --verification-providers=cublas,host --junit-output=test_cutlass_profiler_gemm --print-kernel-before-running=true)
else()
set(CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_GEMM --operation=Gemm --providers=cutlass --verification-providers=cublas,device --junit-output=test_cutlass_profiler_gemm --print-kernel-before-running=true)
set(CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_GEMM --operation=Gemm --providers=cutlass --verification-providers=cublas,device --junit-output=test_cutlass_profiler_gemm --print-kernel-before-running=true)
endif()
set(CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_CONV2D --operation=Conv2d --providers=cutlass --verification-providers=cudnn,device --junit-output=test_cutlass_profiler_conv2d --print-kernel-before-running=true)
tools/profiler/include/cutlass/profiler/conv2d_operation_profiler.h
+1 -1
View File
@@ -430,7 +430,7 @@ public:
protected:
/// Method to profile an initialized CUTLASS operation
virtual Status profile_cutlass_(
double &runtime,
PerformanceResult &result,
Options const &options,
library::Operation const *operation,
void *arguments,
tools/profiler/include/cutlass/profiler/conv3d_operation_profiler.h
+1 -1
View File
@@ -384,7 +384,7 @@ protected:
/// Method to profile an initialized CUTLASS operation
virtual Status profile_cutlass_(
double &runtime,
PerformanceResult &result,
Options const &options,
library::Operation const *operation,
void *arguments,
tools/profiler/include/cutlass/profiler/cublas_helpers.h
+1 -1
View File
@@ -304,7 +304,7 @@ struct cublasLtGemmExDispatcher {
);
/// Executes GEMM using these arguments
cublasStatus_t operator()(cublasLtHandle_t handle);
cublasStatus_t operator()(cublasLtHandle_t handle, cudaStream_t stream = nullptr);
~cublasLtGemmExDispatcher(){
tools/profiler/include/cutlass/profiler/enumerated_types.h
+2 -2
View File
@@ -90,9 +90,9 @@ AlgorithmMode from_string<AlgorithmMode>(std::string const &str);
/// Outcome of a performance test
enum class Disposition {
kPassed,
kFailed,
kFailed, // kernel itself reported an error
kNotRun,
kIncorrect,
kIncorrect, // kernel finished without a detected error, but result does not equal expected result
kNotVerified,
kInvalidProblem,
kNotSupported,
tools/profiler/include/cutlass/profiler/gemm_operation_profiler.h
+6 -3
View File
@@ -143,6 +143,8 @@ public:
/// Buffer used for the cutlass reduction operations' host workspace
std::vector<uint8_t> reduction_host_workspace;
cudaStream_t stream;
};
protected:
@@ -155,7 +157,7 @@ protected:
GemmProblem problem_;
/// Device memory allocations
GemmWorkspace gemm_workspace_;
std::vector<GemmWorkspace> gemm_workspace_;
/// CUTLASS parallel reduction operation to follow this* gemm operation
library::Operation const *reduction_op_;
@@ -231,7 +233,8 @@ protected:
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
ProblemSpace::Problem const &problem,
GemmWorkspace &gemm_workspace);
/// Verifies CUTLASS against host and device references
bool verify_with_reference_(
@@ -246,7 +249,7 @@ protected:
/// Method to profile a CUTLASS Operation
Status profile_cutlass_(
double &runtime,
PerformanceResult &result,
Options const &options,
library::Operation const *operation,
void *arguments,
tools/profiler/include/cutlass/profiler/gpu_timer.h
+11 -6
View File
@@ -51,16 +51,21 @@ struct GpuTimer {
//
GpuTimer();
GpuTimer(GpuTimer const&) = delete;
GpuTimer(GpuTimer &&gpu_timer) noexcept;
~GpuTimer();
/// Records a start event in the stream
void start(cudaStream_t stream = nullptr);
/// Records a start event in the stream, the flag is for cudaEventRecordWithFlags
void start(cudaStream_t stream = nullptr, unsigned int flag = cudaEventRecordDefault);
/// Records a stop event in the stream
void stop(cudaStream_t stream = nullptr);
/// Records a stop event in the stream, the flag is for cudaEventRecordWithFlags
void stop(cudaStream_t stream = nullptr, unsigned int flag = cudaEventRecordDefault);
/// Records a stop event in the stream and synchronizes on the stream
void stop_and_wait(cudaStream_t stream = nullptr);
/// Records a stop event in the stream and synchronizes on the stream, the flag is for cudaEventRecordWithFlags
void stop_and_wait(cudaStream_t stream = nullptr, unsigned int flag = cudaEventRecordDefault);
/// Returns the duration in milliseconds
double duration(int iterations = 1) const;
tools/profiler/include/cutlass/profiler/operation_profiler.h
+1 -1
View File
@@ -232,7 +232,7 @@ protected:
/// Method to profile an initialized CUTLASS operation
virtual Status profile_cutlass_(
double &runtime,
PerformanceResult &result,
Options const &options,
library::Operation const *operation,
void *arguments,
tools/profiler/include/cutlass/profiler/performance_result.h
+3
View File
@@ -86,6 +86,9 @@ struct PerformanceResult {
/// Average runtime in ms
double runtime;
/// Average runtime in ms per device
std::vector<double> runtime_vector;
//
// Members
//
tools/profiler/src/conv2d_operation_profiler.cu
+26 -3
View File
@@ -396,6 +396,29 @@ Status Conv2dOperationProfiler::initialize_configuration(
problem_, operation_desc.conv_kind, operation_desc.A.layout,
operation_desc.B.layout, operation_desc.C.layout);
#if defined(CUTLASS_DEBUG_TRACE_LEVEL) && (CUTLASS_DEBUG_TRACE_LEVEL > 1)
{
auto print_vector = [] (const auto& vec) {
printf("[");
for (size_t k = 0; k < vec.size(); ++k) {
cute::print(vec[k]);
if (k + 1 < vec.size()) {
printf(",");
}
}
printf("]");
};
printf("\n conv_workspace_.configuration.stride_a: ");
print_vector(conv_workspace_.configuration.stride_a);
printf("\n conv_workspace_.configuration.stride_b: ");
print_vector(conv_workspace_.configuration.stride_b);
printf("\n conv_workspace_.configuration.stride_c: ");
print_vector(conv_workspace_.configuration.stride_c);
printf("\n");
}
#endif
// initialize library::ConvArguments
conv_workspace_.arguments.A = nullptr;
conv_workspace_.arguments.B = nullptr;
@@ -1237,7 +1260,7 @@ bool Conv2dOperationProfiler::profile(
}
results_.back().status = profile_cutlass_(
results_.back().runtime,
results_.back(),
options,
operation,
&conv_workspace_.arguments,
@@ -1251,7 +1274,7 @@ bool Conv2dOperationProfiler::profile(
/// Method to profile a CUTLASS Operation
Status Conv2dOperationProfiler::profile_cutlass_(
double &runtime,
PerformanceResult &result,
Options const &options,
library::Operation const *operation,
void *arguments,
@@ -1387,7 +1410,7 @@ Status Conv2dOperationProfiler::profile_cutlass_(
// Update performance result
//
runtime = timer.duration(iteration);
result.runtime = timer.duration(iteration);
return status;
}
tools/profiler/src/conv3d_operation_profiler.cu
+3 -3
View File
@@ -1099,7 +1099,7 @@ bool Conv3dOperationProfiler::profile(
set_cutlass_operator_arguments_();
results_.back().status = profile_cutlass_(
results_.back().runtime,
results_.back(),
options,
operation,
&conv_workspace_.arguments,
@@ -1141,7 +1141,7 @@ void Conv3dOperationProfiler::set_cutlass_operator_arguments_(int problem_idx) {
/// Method to profile a CUTLASS Operation
Status Conv3dOperationProfiler::profile_cutlass_(
double &runtime,
PerformanceResult &result,
Options const &options,
library::Operation const *operation,
void *arguments,
@@ -1248,7 +1248,7 @@ Status Conv3dOperationProfiler::profile_cutlass_(
// Update performance result
//
runtime = timer.duration(iteration);
result.runtime = timer.duration(iteration);
return status;
}
tools/profiler/src/cublas_helpers.cu
+2 -2
View File
@@ -656,7 +656,7 @@ bool cublasLtGemmExDispatcher::get_cublaslt_algo(cublasLtHandle_t handle,
return true;
}
cublasStatus_t cublasLtGemmExDispatcher::operator()(cublasLtHandle_t handle)
cublasStatus_t cublasLtGemmExDispatcher::operator()(cublasLtHandle_t handle, cudaStream_t stream)
{
return cublasLtMatmul(handle,
operationDesc,
@@ -673,7 +673,7 @@ cublasStatus_t cublasLtGemmExDispatcher::operator()(cublasLtHandle_t handle)
&heuristicResult_.algo,
workspace,
heuristicResult_.workspaceSize,
0); //number of streams is set to 0
stream); //number of streams is set to 0
}
tools/profiler/src/device_allocation.cu
+28 -28
View File
@@ -290,9 +290,8 @@ DeviceAllocation::DeviceAllocation():
capacity_(0),
pointer_(nullptr),
layout_(library::LayoutTypeID::kUnknown),
batch_count_(1),
device_(-1) {
batch_count_(1) {
cudaGetDevice(&device_);
}
DeviceAllocation::DeviceAllocation(
@@ -329,13 +328,33 @@ DeviceAllocation::DeviceAllocation(
DeviceAllocation::~DeviceAllocation() {
if (pointer_) {
int current_device;
cudaGetDevice(&current_device);
if (current_device != device_) {
cudaSetDevice(device_);
}
cudaFree(pointer_);
if (current_device != device_) {
cudaSetDevice(current_device);
}
}
}
DeviceAllocation &DeviceAllocation::reset() {
if (pointer_) {
int current_device;
cudaGetDevice(&current_device);
if (current_device != device_) {
cudaSetDevice(device_);
}
cudaFree(pointer_);
if (current_device != device_) {
cudaSetDevice(current_device);
}
}
type_ = library::NumericTypeID::kInvalid;
@@ -2438,25 +2457,11 @@ void DeviceAllocation::fill_host(double val = 0.0) {
cudaError_t DeviceAllocation::malloc(void** ptr, size_t size) {
cudaError_t result;
int set_device_back_to = -1;
int current_device;
cudaGetDevice(&current_device);
/// When needed this sets the device to the allocation's device remembering
/// the current device so that it can be set back after the cudaMalloc is
/// performed.
if (device_ >= 0) {
int current_device;
result = cudaGetDevice(&current_device);
if (result != cudaSuccess) {
return result;
}
if (current_device != device_) {
set_device_back_to = current_device;
result = cudaSetDevice(device_);
if (result != cudaSuccess) {
return result;
}
}
if (current_device != device_) {
cudaSetDevice(device_);
}
// This performs the cudaMalloc
@@ -2465,13 +2470,8 @@ cudaError_t DeviceAllocation::malloc(void** ptr, size_t size) {
return result;
}
/// When needed this sets the device back to what it was when the function was
/// called.
if (set_device_back_to != -1) {
result = cudaSetDevice(set_device_back_to);
if (result != cudaSuccess) {
return result;
}
if (current_device != device_) {
cudaSetDevice(current_device);
}
return cudaSuccess;
tools/profiler/src/gemm_operation_profiler.cu
+522 -407
View File
File diff suppressed because it is too large Load Diff
tools/profiler/src/gpu_timer.cpp
+20 -11
View File
@@ -33,9 +33,11 @@
*/
#include <stdexcept>
#include <cstring>
#include "cutlass/profiler/gpu_timer.h"
namespace cutlass {
namespace profiler {
@@ -52,32 +54,39 @@ GpuTimer::GpuTimer() {
}
}
GpuTimer::GpuTimer(GpuTimer&& gpu_timer) noexcept {
memcpy(events, gpu_timer.events, sizeof(events));
memset(gpu_timer.events, 0, sizeof(gpu_timer.events));
}
GpuTimer::~GpuTimer() {
for (auto & event : events) {
cudaEventDestroy(event);
for (const auto & event : events) {
if (event != nullptr) {
cudaEventDestroy(event);
}
}
}
/// Records a start event in the stream
void GpuTimer::start(cudaStream_t stream) {
cudaError_t result = cudaEventRecord(events[0], stream);
/// Records a start event in the stream, the flag is for cudaEventRecordWithFlags
void GpuTimer::start(cudaStream_t stream, const unsigned int flag) {
cudaError_t result = cudaEventRecordWithFlags(events[0], stream, flag);
if (result != cudaSuccess) {
throw std::runtime_error("Failed to record start event.");
}
}
/// Records a stop event in the stream
void GpuTimer::stop(cudaStream_t stream) {
cudaError_t result = cudaEventRecord(events[1], stream);
/// Records a stop event in the stream, the flag is for cudaEventRecordWithFlags
void GpuTimer::stop(cudaStream_t stream, const unsigned int flag) {
cudaError_t result = cudaEventRecordWithFlags(events[1], stream, flag);
if (result != cudaSuccess) {
throw std::runtime_error("Failed to record stop event.");
}
}
/// Records a stop event in the stream and synchronizes on the stream
void GpuTimer::stop_and_wait(cudaStream_t stream) {
/// Records a stop event in the stream and synchronizes on the stream, the flag is for cudaEventRecordWithFlags
void GpuTimer::stop_and_wait(cudaStream_t stream, const unsigned int flag) {
stop(stream);
stop(stream, flag);
cudaError_t result;
if (stream) {
tools/profiler/src/operation_profiler.cu
+2 -2
View File
@@ -658,7 +658,7 @@ void OperationProfiler::save_workspace(
/// Method to profile a CUTLASS Operation
Status OperationProfiler::profile_cutlass_(
double &runtime,
PerformanceResult &result,
Options const &options,
library::Operation const *operation,
void *arguments,
@@ -726,7 +726,7 @@ Status OperationProfiler::profile_cutlass_(
// Update performance result
//
runtime = timer.duration(iteration);
result.runtime = timer.duration(iteration);
return status;
}
tools/profiler/src/options.cu
+19 -6
View File
@@ -307,12 +307,6 @@ void Options::Initialization::get_distribution(
{0, 0}
};
// Initalize pnz values to a default value of 100%
dist.gaussian.pnz = 1.0;
dist.gaussian.pnzA = 1.0;
dist.gaussian.pnzB = 1.0;
dist.gaussian.pnzC = 1.0;
using KeyValueVector = std::vector<std::pair<std::string, std::string> >;
KeyValueVector values;
@@ -330,6 +324,25 @@ void Options::Initialization::get_distribution(
++it;
}
// Default initialization
switch (dist.kind) {
case cutlass::Distribution::Uniform:
dist.set_uniform(-4/*min*/, 4/*max*/);
break;
case cutlass::Distribution::Gaussian:
dist.set_gaussian(0/*mean*/, 4/*stddev*/);
break;
case cutlass::Distribution::Identity:
dist.set_identity();
break;
case cutlass::Distribution::Sequential:
dist.set_sequential(0/*start*/, 4/*delta*/);
break;
default:
dist.set_uniform(-4/*min*/, 4/*max*/);
return;
}
// Subsequent key-value pairs update the named field of the distribution struct.
for (; it != values.end(); ++it) {
// Integer scaling factor - if < 0, no integer rounding is performed.
tools/profiler/src/performance_report.cpp
+19 -1
View File
@@ -337,7 +337,15 @@ std::ostream & PerformanceReport::print_csv_header_(
<< ",Bytes"
<< ",Flops"
<< ",Flops/Byte"
<< ",Runtime"
<< ",Runtime";
if (options_.device.devices.size() > 1) {
for (size_t i = 0; i < options_.device.devices.size(); i++) {
out << ",Runtime_" << i;
}
}
out
<< ",GB/s"
<< ",GFLOPs"
;
@@ -376,6 +384,16 @@ std::ostream & PerformanceReport::print_result_csv_(
<< "," << result.flops / result.bytes
<< "," << result.runtime;
if (options_.device.devices.size() > 1) {
if (result.runtime_vector.size() != options_.device.devices.size()) {
throw std::runtime_error("Runtime vector size mismatch");
}
for (const auto runtime : result.runtime_vector) {
out << "," << runtime;
}
}
if (result.good()) {
out
tools/profiler/src/rank_2k_operation_profiler.cu
+1 -1
View File
@@ -733,7 +733,7 @@ bool Rank2KOperationProfiler::profile(
rank_k_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
results_.back().status = profile_cutlass_(
results_.back().runtime,
results_.back(),
options,
operation,
&rank_k_workspace_.arguments,
tools/profiler/src/rank_k_operation_profiler.cu
+1 -1
View File
@@ -718,7 +718,7 @@ bool RankKOperationProfiler::profile(
rank_k_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
results_.back().status = profile_cutlass_(
results_.back().runtime,
results_.back(),
options,
operation,
&rank_k_workspace_.arguments,
tools/profiler/src/sparse_gemm_operation_profiler.cu
+1 -1
View File
@@ -578,7 +578,7 @@ bool SparseGemmOperationProfiler::profile(
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
results_.back().status = profile_cutlass_(
results_.back().runtime,
results_.back(),
options,
operation,
&gemm_workspace_.arguments,
tools/profiler/src/symm_operation_profiler.cu
+1 -1
View File
@@ -771,7 +771,7 @@ bool SymmOperationProfiler::profile(
symm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
results_.back().status = profile_cutlass_(
results_.back().runtime,
results_.back(),
options,
operation,
&symm_workspace_.arguments,
tools/profiler/src/trmm_operation_profiler.cu
+1 -1
View File
@@ -709,7 +709,7 @@ bool TrmmOperationProfiler::profile(
trmm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
results_.back().status = profile_cutlass_(
results_.back().runtime,
results_.back(),
options,
operation,
&trmm_workspace_.arguments,
tools/util/include/cutlass/util/device_dump.h
+1 -1
View File
@@ -31,7 +31,7 @@
#pragma once
#include <stdio.h>
#include <cstdio>
#include "cutlass/cutlass.h"
/**
tools/util/include/cutlass/util/device_groupnorm.h
+1 -1
View File
@@ -42,7 +42,7 @@
#include "cutlass/tensor_coord.h"
#include "cutlass/tensor_ref.h"
#include "device_utils.h"
#include <float.h>
#include <cfloat>
namespace cutlass {
tools/util/include/cutlass/util/device_layernorm.h
+1 -1
View File
@@ -42,7 +42,7 @@
#include "cutlass/tensor_coord.h"
#include "cutlass/tensor_ref.h"
#include "device_utils.h"
#include <float.h>
#include <cfloat>
namespace cutlass {
tools/util/include/cutlass/util/device_nhwc_pooling.h
+1 -1
View File
@@ -42,7 +42,7 @@
#include "cutlass/tensor_coord.h"
#include "cutlass/tensor_ref.h"
#include "device_utils.h"
#include <float.h>
#include <cfloat>
namespace cutlass {
tools/util/include/cutlass/util/device_rmsnorm.h
+3 -3
View File
@@ -37,7 +37,7 @@
#include "cutlass/tensor_coord.h"
#include "cutlass/tensor_ref.h"
#include "cutlass/util/device_utils.h"
#include <float.h>
#include <cfloat>
namespace cutlass {
@@ -165,12 +165,12 @@ void rmsnorm(cutlass::MatrixCoord tensor_size,
dim3 grid(m);
if (n % 8 == 0 && std::is_same<T, cutlass::half_t>::value) {
dim3 block(min(1024, (n / 8 + 31) / 32 * 32));
dim3 block(cutlass::platform::min(1024, (n / 8 + 31) / 32 * 32));
rmsnorm_twoPassAlgo_e8<<<grid, block, 0, stream>>>(
(float4 *)output, (const float4 *)input, (const float4 *)weight, m, n, epsilon);
} else {
dim3 block(min(1024, ((n + 31)/32 + 31)/32*32));
dim3 block(cutlass::platform::min(1024, ((n + 31)/32 + 31)/32*32));
rmsnorm_twoPassAlgo_e1<<<grid, block, 0, stream>>>(
output, input, weight, m, n, epsilon);
tools/util/include/cutlass/util/device_utils.h
+1 -1
View File
@@ -36,7 +36,7 @@
#pragma once
#include <cuda_fp16.h>
#include <float.h>
#include <cfloat>
#define FINAL_MASK 0xffffffff
struct half4 {
tools/util/include/cutlass/util/distribution.h
+3
View File
@@ -100,6 +100,9 @@ struct Distribution {
gaussian.mean = _mean;
gaussian.stddev = _stddev;
gaussian.pnz = _pnz;
gaussian.pnzA = _pnz;
gaussian.pnzB = _pnz;
gaussian.pnzC = _pnz;
int_scale = _int_scale;
return *this;
}
tools/util/include/cutlass/util/reference/device/tensor_fill.h
+12 -14
View File
@@ -138,8 +138,8 @@ struct RandomGaussianFunc {
int_scale(int_scale_),
exclude_zero(exclude_zero_) {
float_scale_up = FloatType(IntType(2) << int_scale); // scale up to clamp low order bits
float_scale_down = FloatType(1) / FloatType(IntType(2) << int_scale);
float_scale_up = FloatType(IntType(1) << int_scale); // scale up to clamp low order bits
float_scale_down = FloatType(1) / FloatType(IntType(1) << int_scale);
}
};
@@ -175,8 +175,8 @@ struct RandomGaussianFunc {
Element result;
if (params.int_scale >= 0) {
rnd = FloatType(IntType(std::llround(rnd * params.float_scale_up)));
result = Element(IntType(rnd * params.float_scale_down));
rnd = FloatType(std::llround(rnd * params.float_scale_up));
result = Element(rnd * params.float_scale_down);
}
else {
result = Element(rnd);
@@ -237,7 +237,6 @@ struct RandomGaussianFunc<complex<Real>> {
exclude_zero(exclude_zero_) {
float_scale_up = FloatType(IntType(1) << int_scale);
float_scale_up += FloatType(0.5) * float_scale_up;
float_scale_down = FloatType(1) / FloatType(IntType(1) << int_scale);
}
};
@@ -276,8 +275,8 @@ struct RandomGaussianFunc<complex<Real>> {
Element result;
if (params.int_scale >= 0) {
rnd_r = FloatType(IntType(rnd_r * params.float_scale_up));
rnd_i = FloatType(IntType(rnd_i * params.float_scale_down));
rnd_r = FloatType(std::llround(rnd_r * params.float_scale_up));
rnd_i = FloatType(std::llround(rnd_i * params.float_scale_up));
result = {
Real(rnd_r * params.float_scale_down),
@@ -482,8 +481,8 @@ struct RandomUniformFunc {
pnan(pnan_),
exclude_zero(exclude_zero_) {
float_scale_up = FloatType(IntType(2) << int_scale); // scale up to clamp low order bits
float_scale_down = FloatType(1) / FloatType(IntType(2) << int_scale);
float_scale_up = FloatType(IntType(1) << int_scale); // scale up to clamp low order bits
float_scale_down = FloatType(1) / FloatType(IntType(1) << int_scale);
// Handle cases where min = 0 or max = 0 for excluding zeros
if (exclude_zero >= 0) {
@@ -535,8 +534,8 @@ struct RandomUniformFunc {
Element result;
if (params.int_scale >= 0) {
rnd = FloatType(IntType(std::llround(rnd * params.float_scale_up)));
result = Element(IntType(rnd * params.float_scale_down));
rnd = FloatType(std::llround(rnd * params.float_scale_up));
result = Element(rnd * params.float_scale_down);
}
else {
result = Element(rnd);
@@ -612,7 +611,6 @@ struct RandomUniformFunc<complex<Real>> {
exclude_zero(exclude_zero_) {
float_scale_up = FloatType(IntType(1) << int_scale);
float_scale_up += FloatType(0.5) * float_scale_up;
float_scale_down = FloatType(1) / FloatType(IntType(1) << int_scale);
// Handle cases where min = 0 or max = 0 for excluding zeros
@@ -668,8 +666,8 @@ struct RandomUniformFunc<complex<Real>> {
Element result;
if (params.int_scale >= 0) {
rnd_r = FloatType(IntType(rnd_r * params.float_scale_up));
rnd_i = FloatType(IntType(rnd_i * params.float_scale_up));
rnd_r = FloatType(std::llround(rnd_r * params.float_scale_up));
rnd_i = FloatType(std::llround(rnd_i * params.float_scale_up));
result = {
Real(rnd_r * params.float_scale_down),
tools/util/include/cutlass/util/reference/host/gett.hpp
-1
View File
@@ -281,7 +281,6 @@ void gett_epilogue(
cute::is_same_v<ElementAux, cutlass::uint1b_t>;
constexpr bool IsClamp =
cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::Clamp<ElementCompute>>;
constexpr bool IsBackpropFusion =
cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::dGELU<ElementCompute>> or
cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::dReLU<ElementCompute>>;
tools/util/include/cutlass/util/reference/host/rank_2k_complex.h
+1 -1
View File
@@ -41,7 +41,7 @@
#include "cutlass/numeric_conversion.h"
#include "cutlass/tensor_view.h"
#include "cutlass/gemm/gemm.h"
#include <assert.h>
#include <cassert>
namespace cutlass {
namespace reference {
tools/util/include/cutlass/util/reference/host/rank_k_complex.h
+1 -1
View File
@@ -41,7 +41,7 @@
#include "cutlass/numeric_conversion.h"
#include "cutlass/tensor_view.h"
#include "cutlass/gemm/gemm.h"
#include <assert.h>
#include <cassert>
namespace cutlass {
namespace reference {
tools/util/include/cutlass/util/reference/host/symm_complex.h
+1 -1
View File
@@ -41,7 +41,7 @@
#include "cutlass/numeric_conversion.h"
#include "cutlass/tensor_view.h"
#include "cutlass/gemm/gemm.h"
#include <assert.h>
#include <cassert>
namespace cutlass {
namespace reference {
tools/util/include/cutlass/util/reference/host/tensor_fill.h
+8 -8
View File
@@ -269,8 +269,8 @@ struct RandomGaussianFunc<complex<Element> > {
// Sample from the Gaussian distribution for a nonzero element
if (bernoulli_result) {
if (int_scale >= 0) {
rnd[0] = double(int(rnd[0] * double(1 << int_scale)));
rnd[1] = double(int(rnd[1] * double(1 << int_scale)));
rnd[0] = double(std::llround(rnd[0] * double(1 << int_scale)));
rnd[1] = double(std::llround(rnd[1] * double(1 << int_scale)));
reals[0] = from_real<Element>(rnd[0] / double(1 << int_scale));
reals[1] = from_real<Element>(rnd[1] / double(1 << int_scale));
}
@@ -348,10 +348,10 @@ struct RandomGaussianFunc<Quaternion<Element> > {
// Sample from the Gaussian distribution for a nonzero element
if (bernoulli_result) {
if (int_scale >= 0) {
rnd1[0] = double(int(rnd1[0] * double(1 << int_scale)));
rnd1[1] = double(int(rnd1[1] * double(1 << int_scale)));
rnd2[0] = double(int(rnd2[0] * double(1 << int_scale)));
rnd2[1] = double(int(rnd2[1] * double(1 << int_scale)));
rnd1[0] = double(std::llround(rnd1[0] * double(1 << int_scale)));
rnd1[1] = double(std::llround(rnd1[1] * double(1 << int_scale)));
rnd2[0] = double(std::llround(rnd2[0] * double(1 << int_scale)));
rnd2[1] = double(std::llround(rnd2[1] * double(1 << int_scale)));
reals[0] = from_real<Element>(rnd1[0] / double(1 << int_scale));
reals[1] = from_real<Element>(rnd1[1] / double(1 << int_scale));
@@ -725,7 +725,7 @@ public:
// testing
if (int_scale >= 0) {
rnd = double(int(rnd * double(1 << int_scale)));
rnd = double(std::llround(rnd * double(1 << int_scale)));
reals[i] = from_real<Element>(Real(rnd / double(1 << int_scale)));
}
else {
@@ -808,7 +808,7 @@ public:
// testing
if (int_scale >= 0) {
rnd = double(int(rnd * double(1 << int_scale)));
rnd = double(std::llround(rnd * double(1 << int_scale)));
reals[i] = from_real<Element>(Real(rnd / double(1 << int_scale)));
}
else {
tools/util/include/cutlass/util/type_traits.h
+1 -1
View File
@@ -36,7 +36,7 @@
#include <cublas_v2.h>
#include <cuda_fp16.h>
#include <stdint.h>
#include <cstdint>
#include "cutlass/numeric_types.h"
#include "cutlass/complex.h"