leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

releaase 2.11 (#703)

Browse Source
This commit is contained in:
Aditya Atluri
2022-11-19 06:02:15 -08:00
committed by GitHub
parent 3c90f6aea6
commit c975e2ccbb
329 changed files with 47332 additions and 10607 deletions
Download Patch File Download Diff File Expand all files Collapse all files

261
tools/library/src/conv2d_operation.h
View File

@@ -36,9 +36,12 @@
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/conv/kernel/default_conv2d_fprop.h"
#include "cutlass/conv/kernel/default_conv2d_group_fprop.h"
#include "cutlass/conv/kernel/default_depthwise_fprop.h"
#include "cutlass/conv/kernel/default_conv2d_dgrad.h"
#include "cutlass/conv/kernel/default_conv2d_wgrad.h"
#include "cutlass/conv/device/implicit_gemm_convolution.h"
#include "cutlass/conv/device/direct_convolution.h"
#include "cutlass/library/library.h"
#include "library_internal.h"
@@ -98,9 +101,9 @@ public:
description_.tile_description.threadblock_stages = Operator::kStages;
description_.tile_description.warp_count = make_Coord(
Operator::ImplicitGemmKernel::WarpCount::kM,
Operator::ImplicitGemmKernel::WarpCount::kN,
Operator::ImplicitGemmKernel::WarpCount::kK);
Operator::UnderlyingKernel::WarpCount::kM,
Operator::UnderlyingKernel::WarpCount::kN,
Operator::UnderlyingKernel::WarpCount::kK);
description_.tile_description.math_instruction.instruction_shape = make_Coord(
Operator::InstructionShape::kM,
@@ -381,6 +384,258 @@ public:
}
};
///////////////////////////////////////////////////////////////////////////////////////////////////
//
// DirectConv2d library operation class for cutlass profiler
//
///////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Operator_>
class DirectConv2dOperation : public Conv2dOperation<Operator_> {
public:
using Operator = Operator_;
using Base = Conv2dOperation<Operator_>;
using ElementA = typename Operator::ElementA;
using LayoutA = typename Operator::LayoutA;
using ElementB = typename Operator::ElementB;
using LayoutB = typename Operator::LayoutB;
using ElementC = typename Operator::ElementC;
using LayoutC = typename Operator::LayoutC;
using ElementAccumulator = typename Operator::ElementAccumulator;
using ElementCompute = typename Operator::EpilogueOutputOp::ElementCompute;
static cutlass::conv::Operator const kConvolutionalOperator = Operator::kConvolutionalOperator;
using OperatorArguments = typename Operator::Arguments;
public:
/// Constructor
DirectConv2dOperation(char const *name = "unknown_direct)conv2d_fprop") : Conv2dOperation<Operator_>(name) {
this->description_.conv_kind = ConvKindMap<kConvolutionalOperator>::kId;
}
protected:
/// Constructs the arguments structure given the configuration and arguments
static Status construct_arguments_(
OperatorArguments &operator_args,
Conv2dConfiguration const *configuration) {
operator_args.problem_size = configuration->problem_size;
operator_args.ref_A =
{
nullptr,
LayoutA::packed(implicit_gemm_tensor_a_extent(kConvolutionalOperator, configuration->problem_size))
};
operator_args.ref_B =
{
nullptr,
LayoutB::packed(implicit_gemm_tensor_b_extent(kConvolutionalOperator, configuration->problem_size))
};
operator_args.ref_reordered_B =
{
nullptr,
LayoutB::packed(implicit_gemm_tensor_b_extent(kConvolutionalOperator, configuration->problem_size))
};
operator_args.ref_C =
{
nullptr,
LayoutC::packed(implicit_gemm_tensor_c_extent(kConvolutionalOperator, configuration->problem_size))
};
operator_args.ref_D =
{
nullptr,
LayoutC::packed(implicit_gemm_tensor_c_extent(kConvolutionalOperator, configuration->problem_size))
};
operator_args.split_k_mode = configuration->split_k_mode;
return Status::kSuccess;
}
/// Constructs the arguments structure given the configuration and arguments
static Status update_arguments_(
OperatorArguments &operator_args,
ConvArguments const *arguments) {
if (arguments->pointer_mode == ScalarPointerMode::kHost) {
typename Operator::EpilogueOutputOp::Params params(
*static_cast<ElementCompute const *>(arguments->alpha),
*static_cast<ElementCompute const *>(arguments->beta)
);
operator_args.output_op = params;
}
else if (arguments->pointer_mode == ScalarPointerMode::kDevice){
typename Operator::EpilogueOutputOp::Params params(
static_cast<ElementCompute const *>(arguments->alpha),
static_cast<ElementCompute const *>(arguments->beta)
);
operator_args.output_op = params;
}
else {
return Status::kErrorInvalidProblem;
}
operator_args.ref_A.reset(static_cast<ElementA *>(const_cast<void *>(arguments->A)));
operator_args.ref_B.reset(static_cast<ElementB *>(const_cast<void *>(arguments->B)));
operator_args.ref_C.reset(static_cast<ElementC *>(const_cast<void *>(arguments->C)));
operator_args.ref_D.reset(static_cast<ElementC *>(const_cast<void *>(arguments->D)));
operator_args.ref_reordered_B.reset(static_cast<ElementC *>(const_cast<void *>(arguments->reordered_B)));
return Status::kSuccess;
}
public:
/// Returns success if the operation can proceed
virtual Status can_implement(
void const *configuration_ptr,
void const *arguments_ptr) const {
Conv2dConfiguration const *configuration =
static_cast<Conv2dConfiguration const *>(configuration_ptr);
ConvArguments const *arguments =
static_cast<ConvArguments const *>(arguments_ptr);
OperatorArguments args;
Status status = construct_arguments_(args, configuration);
if (status != Status::kSuccess) {
return status;
}
status = update_arguments_(args, arguments);
if (status != Status::kSuccess) {
return status;
}
return Operator::can_implement(args);
}
/// Gets the host-side workspace
virtual uint64_t get_host_workspace_size(
void const *configuration) const {
return sizeof(Operator);
}
/// Gets the device-side workspace
virtual uint64_t get_device_workspace_size(
void const *configuration_ptr,
void const *arguments_ptr = nullptr) const {
OperatorArguments args;
Status status = construct_arguments_(
args,
static_cast<Conv2dConfiguration const *>(configuration_ptr));
if (status != Status::kSuccess) {
return 0;
}
return Operator::get_workspace_size(args);
}
/// Initializes the workspace
virtual Status initialize(
void const *configuration_ptr,
void *host_workspace,
void *device_workspace,
cudaStream_t stream = nullptr) const {
OperatorArguments args;
Status status = construct_arguments_(
args,
static_cast<Conv2dConfiguration const *>(configuration_ptr));
if (status != Status::kSuccess) {
return status;
}
Operator *op = new (host_workspace) Operator;
//std::cout << "initialize library::Conv2dOperation" << std::endl;
//print_operator_args(args);
return op->initialize(args, device_workspace, stream);
}
/// Runs the kernel
virtual Status run(
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const {
OperatorArguments args;
Status status = update_arguments_(
args,
static_cast<ConvArguments const *>(arguments_ptr));
if (status != Status::kSuccess) {
return status;
}
Operator *op = static_cast<Operator *>(host_workspace);
status = op->update(args, device_workspace);
if (status != Status::kSuccess) {
return status;
}
//std::cout << "run library::Conv2dOperation" << std::endl;
//print_operator_args(args);
return op->run(stream);
}
/// Call print_operator_args from the Conv2dOperation::initialize()
// to dump arguments passed on to cutlass operator for debugging
void print_operator_args(OperatorArguments &operator_args) const {
std::cout << "Conv2dOperation::OperatorArguments" << std::endl
<< " problem_size:" << std::endl
<< operator_args.problem_size << std::endl
<< " split_k_mode: "
<< (operator_args.split_k_mode == cutlass::conv::SplitKMode::kSerial ? "serial" : "parallel") << std::endl
<< " epilouge (alpha, beta): "
<< operator_args.output_op.alpha << ", "
<< operator_args.output_op.beta << std::endl
<< " ref_A (ptr, {stride}): "
<< operator_args.ref_A.data() << ", {"
<< operator_args.ref_A.stride(0) << ", "
<< operator_args.ref_A.stride(1) << ", "
<< operator_args.ref_A.stride(2) << "}" << std::endl
<< " ref_B (ptr, {stride}): "
<< operator_args.ref_B.data() << ", {"
<< operator_args.ref_B.stride(0) << ", "
<< operator_args.ref_B.stride(1) << ", "
<< operator_args.ref_B.stride(2) << "}" << std::endl
<< " ref_C (ptr, {stride}): "
<< operator_args.ref_C.data() << ", {"
<< operator_args.ref_C.stride(0) << ", "
<< operator_args.ref_C.stride(1) << ", "
<< operator_args.ref_C.stride(2) << "}" << std::endl
<< " ref_D (ptr, {stride}): "
<< operator_args.ref_D.data() << ", {"
<< operator_args.ref_D.stride(0) << ", "
<< operator_args.ref_D.stride(1) << ", "
<< operator_args.ref_D.stride(2) << "}" << std::endl;
}
};
} // namespace library
} // namespace cutlass

6
tools/library/src/conv3d_operation.h
View File

@@ -98,9 +98,9 @@ public:
description_.tile_description.threadblock_stages = Operator::kStages;
description_.tile_description.warp_count = make_Coord(
Operator::ImplicitGemmKernel::WarpCount::kM,
Operator::ImplicitGemmKernel::WarpCount::kN,
Operator::ImplicitGemmKernel::WarpCount::kK);
Operator::UnderlyingKernel::WarpCount::kM,
Operator::UnderlyingKernel::WarpCount::kN,
Operator::UnderlyingKernel::WarpCount::kK);
description_.tile_description.math_instruction.instruction_shape = make_Coord(
Operator::InstructionShape::kM,

30
tools/library/src/gemm_operation.h
View File

@@ -305,7 +305,7 @@ public:
Operator *op = static_cast<Operator *>(host_workspace);
status = op->update(args, device_workspace);
status = op->update(args);
if (status != Status::kSuccess) {
return status;
@@ -507,7 +507,7 @@ public:
Operator *op = static_cast<Operator *>(host_workspace);
status = op->update(args, device_workspace);
status = op->update(args);
if (status != Status::kSuccess) {
return status;
@@ -725,8 +725,8 @@ public:
}
Operator *op = static_cast<Operator *>(host_workspace);
status = op->update(args, device_workspace);
status = op->update(args);
if (status != Status::kSuccess) {
return status;
@@ -917,30 +917,30 @@ public:
/// Runs the kernel
virtual Status run(
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const {
OperatorArguments args;
Status status = update_arguments_(
args,
args,
static_cast<GemmPlanarComplexArguments const *>(arguments_ptr));
if (status != Status::kSuccess) {
return status;
}
Operator *op = static_cast<Operator *>(host_workspace);
status = op->update(args, device_workspace);
status = op->update(args);
if (status != Status::kSuccess) {
return status;
}
status = op->run(stream);
return status;
}
};
@@ -1134,7 +1134,7 @@ public:
Operator *op = static_cast<Operator *>(host_workspace);
status = op->update(args, device_workspace);
status = op->update(args);
if (status != Status::kSuccess) {
return status;
@@ -1336,7 +1336,7 @@ public:
Operator *op = static_cast<Operator *>(host_workspace);
status = op->update(args, device_workspace);
status = op->update(args);
if (status != Status::kSuccess) {
return status;