leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

CUTLASS 3.3.0 (#1167)

Browse Source 
* Release 3.3.0

Adds support for mixed precision GEMMs On Hopper and Ampere
Adds support for < 16B aligned GEMMs on Hopper
Enhancements to EVT
Enhancements to Python interface
Enhancements to Sub-byte type handling in CuTe
Several other bug-fixes and performance improvements.

* minor doc update
This commit is contained in:
Pradeep Ramani
2023-11-02 08:09:05 -07:00
committed by GitHub
parent 922fb5108b
commit c008b4aea8
263 changed files with 16214 additions and 5008 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tools/library/CMakeLists.txt
View File

@@ -264,7 +264,8 @@ set(CUTLASS_LIBRARY_GENERATED_KERNEL_LIST_FILE ${CMAKE_CURRENT_BINARY_DIR}/gener
# in ${CMAKE_CURRENT_BINARY_DIR}/library_instance_generation.log, set this parameter to INFO
execute_process(
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../../python/cutlass_library
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/../../python/cutlass_library/generator.py
COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CUTLASS_LIBRARY_PACKAGE_DIR}
${Python3_EXECUTABLE} ${CUTLASS_SOURCE_DIR}/python/cutlass_library/generator.py
--operations "${CUTLASS_LIBRARY_OPERATIONS}"
--build-dir ${PROJECT_BINARY_DIR}
--curr-build-dir ${CMAKE_CURRENT_BINARY_DIR}
@@ -275,6 +276,7 @@ execute_process(
--selected-kernel-list "${CUTLASS_LIBRARY_GENERATED_KERNEL_LIST_FILE}"
--cuda-version "${CUTLASS_GENERATOR_CUDA_COMPILER_VERSION}"
--log-level DEBUG
--disable-cutlass-package-imports
RESULT_VARIABLE cutlass_lib_INSTANCE_GENERATION_RESULT
OUTPUT_VARIABLE cutlass_lib_INSTANCE_GENERATION_OUTPUT
OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/library_instance_generation.log

7
tools/library/include/cutlass/library/operation_table.h
View File

@@ -215,21 +215,21 @@ struct GemmPreferenceKey {
return compute_capability == rhs.compute_capability;
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
inline
std::ostream& operator<< (std::ostream& out, const cutlass::library::GemmPreferenceKey& key) {
out << "{\n"
<< "compute_capability : " << key.compute_capability << std::endl
<< "alignment : " << key.alignment << std::endl
<< "}";
return out;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Maps minimum compute capability onto a vector of possible operations
using GemmOperationVectorMap = std::map<
GemmPreferenceKey,
@@ -242,7 +242,6 @@ using GemmOperationFunctionalMap = std::unordered_map<
GemmOperationVectorMap,
GemmFunctionalKeyHasher
>;
/////////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////////////
// Data Structures for Conv Functional Maps

2
tools/library/src/handle.cu
View File

@@ -1183,7 +1183,7 @@ Operation const* find_gemm_operation_for_parallel_reduction(Operation const *ope
return nullptr;
}
// return matching gemm operation (same tile shape, stages, warp count, and instruction)
// return matching gemm opertion (same tile shape, stages, warp count, and instruction)
for (auto op : it->second) {
if (op->description().tile_description == operation->description().tile_description) {
return op;

4
tools/library/src/library_internal.h
View File

@@ -287,6 +287,10 @@ template <> struct OpcodeClassMap<arch::OpClassTensorOp> {
static OpcodeClassID const kId = OpcodeClassID::kTensorOp;
};
template <> struct OpcodeClassMap<arch::OpClassSparseTensorOp> {
static OpcodeClassID const kId = OpcodeClassID::kSparseTensorOp;
};
template <> struct OpcodeClassMap<arch::OpClassWmmaTensorOp> {
static OpcodeClassID const kId = OpcodeClassID::kWmmaTensorOp;
};

2
tools/library/src/operation_table.cu
View File

@@ -47,9 +47,7 @@ void OperationTable::append(Manifest const &manifest) {
// Insert operations into appropriate data structure
for (auto const & operation : manifest) {
OperationDescription const &desc = operation->description();
// insert all gemm operation into operation table
if (desc.kind == OperationKind::kGemm) {
GemmDescription const &gemm_desc = static_cast<GemmDescription const &>(desc);

1
tools/library/src/reduction/init_reduction_operations.cu
View File

@@ -42,6 +42,7 @@ namespace library {
///////////////////////////////////////////////////////////////////////////////////////////////
// CUTLASS Reduction Instances //
///////////////////////////////////////////////////////////////////////////////////////////////
void initialize_reduce_add_linear_combination_f16_f16_f16(Manifest &manifest);
void initialize_reduce_add_linear_combination_f32_f32_f16(Manifest &manifest);
void initialize_reduce_add_linear_combination_f32_f32_bf16(Manifest &manifest);

1
tools/library/src/reduction/reduction_device.cu
View File

@@ -146,7 +146,6 @@ void initialize_reduce_add_linear_combination_f32_f32_bf16(Manifest &manifest) {
));
}
void initialize_reduce_add_linear_combination_f32_f32_f32(Manifest &manifest) {
using ElementWorkspace = float;

3
tools/library/src/reference/gemm_fp_mixed_input.cu
View File

@@ -78,7 +78,7 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
uint8_t,
@@ -151,4 +151,3 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
} // namespace cutlass
///////////////////////////////////////////////////////////////////////////////////////////////////

1
tools/library/src/reference/initialize_reference_operations.cu
View File

@@ -84,6 +84,7 @@ void initialize_reference_operations(Manifest &manifest) {
initialize_gemm_reference_operations_fp32out(manifest);
initialize_gemm_reference_operations_fp_other(manifest);
initialize_gemm_reference_operations_fp_mixed_input(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

2
tools/library/src/util.cu
View File

@@ -333,7 +333,7 @@ static struct {
}
OperationKind_enumerants[] = {
{"eq_gemm", "EqGemm", OperationKind::kEqGemm},
{"gemm", "Gemm", OperationKind::kGemm},
{"gemm", "Gemm", OperationKind::kGemm},
{"rank_k", "RankK", OperationKind::kRankK},
{"rank_2k", "Rank2K", OperationKind::kRank2K},
{"trmm", "Trmm", OperationKind::kTrmm},

7
tools/profiler/CMakeLists.txt
View File

@@ -97,7 +97,12 @@ install(
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
)
set(CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_GEMM --operation=Gemm --providers=cutlass --verification-providers=cublas,device --junit-output=test_cutlass_profiler_gemm --print-kernel-before-running=true)
if (CUDA_VERSION VERSION_GREATER_EQUAL 12.3 AND CUDA_VERSION VERSION_LESS 12.4 AND (90a IN_LIST CUTLASS_NVCC_ARCHS_ENABLED OR (90 IN_LIST CUTLASS_NVCC_ARCHS_ENABLED)))
set(CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_GEMM --operation=Gemm --providers=cutlass --verification-providers=cublas,host --junit-output=test_cutlass_profiler_gemm --print-kernel-before-running=true)
else()
set(CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_GEMM --operation=Gemm --providers=cutlass --verification-providers=cublas,device --junit-output=test_cutlass_profiler_gemm --print-kernel-before-running=true)
endif()
set(CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_CONV2D --operation=Conv2d --providers=cutlass --verification-providers=cudnn,device --junit-output=test_cutlass_profiler_conv2d --print-kernel-before-running=true)
set(CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_CONV3D --operation=Conv3d --providers=cutlass --verification-providers=cudnn,device,host --junit-output=test_cutlass_profiler_conv3d --print-kernel-before-running=true)
set(CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_SPGEMM --operation=SparseGemm --providers=cutlass --verification-providers=cublas,device,host --junit-output=test_cutlass_profiler_spgemm --print-kernel-before-running=true)

1
tools/profiler/src/device_allocation.cu
View File

@@ -2005,7 +2005,6 @@ void DeviceAllocation::write_tensor_csv(
case library::NumericTypeID::kFE5M2:
write_tensor_csv_static_type<float_e5m2_t>(out, *this);
break;
case library::NumericTypeID::kF16:
write_tensor_csv_static_type<half_t>(out, *this);
break;

2
tools/profiler/src/gemm_operation_profiler.cu
View File

@@ -585,11 +585,11 @@ Status GemmOperationProfiler::initialize_workspace(
workspace_size = underlying_operation->get_device_workspace_size(&gemm_workspace_.configuration,
&gemm_workspace_.arguments);
gemm_workspace_.device_workspace.reset(library::NumericTypeID::kU8, workspace_size);
status = underlying_operation->initialize(
&gemm_workspace_.configuration,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data());
if (status != Status::kSuccess) {
return status;
}

2
tools/util/include/cutlass/util/device_rmsnorm.h
View File

@@ -118,7 +118,7 @@ __global__ void rmsnorm_twoPassAlgo_e1(T* output,
const T* input,
const T* weight,
const int m, const int n,
float epsilon)
float epsilon)
{
const int m_idx = blockIdx.x;
const int tid = threadIdx.x;

13
tools/util/include/cutlass/util/host_tensor.h
View File

@@ -112,7 +112,7 @@ public:
/// Example
/// int2: kBitsStoredVec = 8; kElementsPerStoredVec = 4; kNumStoragePerStoredVec = 1 uint8_t;
/// int4: kBitsStoredVec = 8; kElementsPerStoredVec = 2; kNumStoragePerStoredVec = 1 uint8_t;
static int const kBitsStoredVec = (sizeof_bits<Element>::value < 8) ? cutlass::lcm(sizeof_bits<Element>::value, 8) : sizeof_bits<Element>::value;
static int const kBitsStoredVec = (sizeof_bits<Element>::value < 8) ? cutlass::lcm(static_cast<int>(sizeof_bits<Element>::value), 8) : sizeof_bits<Element>::value;
static int const kElementsPerStoredVec = kBitsStoredVec / sizeof_bits<Element>::value;
static int const kNumStoragePerStoredVec = kBitsStoredVec / (sizeof(Element) * 8);
@@ -129,7 +129,8 @@ public:
Layout layout_;
/// Host-side memory allocation
std::vector<Element> host_;
/// avoid the std::vector<bool> specialization
std::vector<std::conditional_t<std::is_same_v<Element,bool>, uint8_t, Element>> host_;
/// Device-side memory
device_memory::allocation<Element> device_;
@@ -250,10 +251,10 @@ public:
}
/// Gets pointer to host data
Element * host_data() { return host_.data(); }
Element * host_data() { return reinterpret_cast<Element *>(host_.data()); }
/// Gets pointer to host data with a pointer offset
Element * host_data_ptr_offset(LongIndex ptr_element_offset) { return &ReferenceFactory<Element>::get(host_.data(), ptr_element_offset); }
Element * host_data_ptr_offset(LongIndex ptr_element_offset) { return &ReferenceFactory<Element>::get(host_data(), ptr_element_offset); }
/// Gets a reference to an element in host memory
Reference host_data(LongIndex idx) {
@@ -261,10 +262,10 @@ public:
}
/// Gets pointer to host data
Element const * host_data() const { return host_.data(); }
Element const * host_data() const { return reinterpret_cast<Element const *>(host_.data()); }
/// Gets pointer to host data with a pointer offset
Element const * host_data_ptr_offset(LongIndex ptr_element_offset) const { return &ReferenceFactory<Element>::get(host_.data(), ptr_element_offset); }
Element const * host_data_ptr_offset(LongIndex ptr_element_offset) const { return &ReferenceFactory<Element>::get(host_data(), ptr_element_offset); }
/// Gets a constant reference to an element in host memory
ConstReference host_data(LongIndex idx) const {

5
tools/util/include/cutlass/util/print_error.hpp
View File

@@ -63,6 +63,7 @@ matrix_inf_norm(cute::Tensor<EngineType, LayoutType> const& host_matrix)
{
using std::abs;
using error_type = decltype(std::declval<matrix_inf_norm_result>().inf_norm);
using element_type = typename EngineType::value_type;
error_type inf_norm = 0.0;
bool found_nan = false;
@@ -95,6 +96,7 @@ matrix_diff_inf_norm(cute::Tensor<EngineType, LayoutType> const& X,
{
using std::abs;
using error_type = decltype(std::declval<matrix_inf_norm_result>().inf_norm);
using element_type = typename EngineType::value_type;
assert(cute::size<0>(X) == cute::size<0>(Y));
assert(cute::size<1>(X) == cute::size<1>(Y));
@@ -110,7 +112,8 @@ matrix_diff_inf_norm(cute::Tensor<EngineType, LayoutType> const& X,
for(int64_t i = 0; i < num_rows; ++i) {
error_type row_abs_sum = 0.0;
for(int64_t j = 0; j < num_cols; ++j) {
row_abs_sum += abs(X(i,j) - Y(i,j));
row_abs_sum += error_type(abs(element_type(X(i,j)) -
element_type(Y(i,j))));
}
if(std::isnan(row_abs_sum)) {
found_nan = true;

84
tools/util/include/cutlass/util/reference/host/gett.hpp
View File

@@ -46,6 +46,16 @@
namespace cutlass::reference::host {
template<class T, class = void>
struct ElementTraits {
using type = T;
};
template<class T>
struct ElementTraits<T, std::enable_if_t<!std::is_same_v<decltype(std::declval<T>().get()), void> > > {
using type = decltype(std::declval<T>().get());
};
/////////////////////////////////////////////////////////////////////////////////////////////////
template<
@@ -170,8 +180,8 @@ void gett_mainloop(
static_assert(cute::rank(typename MainloopParams::LayoutA{}) == 3, "M, K, B");
static_assert(cute::rank(typename MainloopParams::LayoutB{}) == 3, "N, K, B");
using ElementA = typename MainloopParams::TensorA::value_type;
using ElementB = typename MainloopParams::TensorB::value_type;
using ElementA = typename ElementTraits<typename MainloopParams::EngineA::value_type>::type;
using ElementB = typename ElementTraits<typename MainloopParams::EngineB::value_type>::type;
using RingOp = multiply_add<ElementAccumulator, ElementAccumulator, ElementAccumulator>;
RingOp fma_op;
@@ -189,7 +199,8 @@ void gett_mainloop(
ElementAccumulator a_frag[kBlockM];
for (int m_b = 0; m_b < kBlockM; ++m_b) {
if (m + m_b < cute::size<0>(mainloop_params.A.layout())) {
a_frag[m_b] = static_cast<ElementAccumulator>(mainloop_params.A(m + m_b, k, l));
// Perform reference GEMM calculations at the accumulator's precision. Cast A value to accumulator type.
a_frag[m_b] = static_cast<ElementAccumulator>(ElementA(mainloop_params.A(m + m_b, k, l)));
if (mainloop_params.transform_A == ComplexTransform::kConjugate) {
a_frag[m_b] = conj(a_frag[m_b]);
}
@@ -202,7 +213,8 @@ void gett_mainloop(
ElementAccumulator b_frag[kBlockN];
for (int n_b = 0; n_b < kBlockN; ++n_b) {
if (n + n_b < cute::size<0>(mainloop_params.B.layout())) {
b_frag[n_b] = static_cast<ElementAccumulator>(mainloop_params.B(n + n_b, k, l));
// Perform reference GEMM calculations at the accumulator's precision. Cast A value to accumulator type.
b_frag[n_b] = static_cast<ElementAccumulator>(ElementB(mainloop_params.B(n + n_b, k, l)));
if (mainloop_params.transform_B == ComplexTransform::kConjugate) {
b_frag[n_b] = conj(b_frag[n_b]);
}
@@ -252,10 +264,19 @@ void gett_epilogue(
std::is_same_v<ElementAux, cutlass::float_e4m3_t> or
std::is_same_v<ElementAux, cutlass::float_e5m2_t>;
constexpr bool IsReLUAuxNeeded =
cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::ReLu<ElementCompute>> and
cute::is_same_v<ElementAux, cutlass::uint1b_t>;
constexpr bool IsBackpropFusion =
cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::dGELU<ElementCompute>> or
cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::dReLU<ElementCompute>>;
// Input related converter
NumericConverter<ElementCompute, ElementAccumulator> accumulator_converter;
NumericConverter<ElementCompute, ElementC> source_converter;
NumericConverter<ElementCompute, ElementBias> bias_converter;
NumericConverter<ElementCompute, ElementAux> aux_source_converter;
// Scale related converter
NumericConverter<ElementCompute, ElementScalar> scale_converter;
@@ -267,10 +288,12 @@ void gett_epilogue(
// Output related converter
NumericConverter<ElementD, ElementCompute> destination_converter;
NumericConverter<ElementAux, ElementCompute> aux_destination_converter;
NumericConverter<ElementBias, ElementCompute> dBias_converter;
// Epilogue operations
multiply_add<ElementCompute, ElementCompute, ElementCompute> epilogue_fma;
multiplies<ElementCompute> mul;
plus<ElementCompute> add;
// Activation operation
ActivationFunctor activation;
@@ -294,23 +317,25 @@ void gett_epilogue(
converted_alpha = mul(converted_alpha, mul(converted_scale_a, converted_scale_b));
converted_beta = mul(converted_beta, converted_scale_c);
for (int n_b = 0; n_b < kBlockN; ++n_b) {
for (int m_b = 0; m_b < kBlockM; ++m_b) {
for (int m_b = 0; m_b < kBlockM; ++m_b) {
ElementCompute local_dBias = ElementCompute(0);
for (int n_b = 0; n_b < kBlockN; ++n_b) {
if (m + m_b < cute::size<0>(epilogue_params.D.layout()) && n + n_b < cute::size<1>(epilogue_params.D.layout())) {
// Convert every type to ElementCompute first, do compute, convert to output type, write it out
ElementCompute converted_acc = accumulator_converter(acc[m_b][n_b]);
// per-row alpha
if (epilogue_params.Valpha.data()) {
if (raw_pointer_cast(epilogue_params.Valpha.data())) {
converted_alpha = scale_converter(epilogue_params.Valpha(m + m_b));
}
ElementCompute output = mul(converted_alpha, converted_acc);
if (epilogue_params.Bias.data()) {
if (raw_pointer_cast(epilogue_params.Bias.data()) && not IsBackpropFusion) {
ElementCompute converted_bias = bias_converter(epilogue_params.Bias(m + m_b));
output = bias_op(output, converted_bias);
}
if (epilogue_params.C.data()) {
if (raw_pointer_cast(epilogue_params.C.data())) {
ElementCompute converted_src = source_converter(epilogue_params.C(m + m_b, n + n_b, l));
// per-row beta
if (epilogue_params.Vbeta.data()) {
@@ -319,18 +344,33 @@ void gett_epilogue(
output = epilogue_fma(converted_beta, converted_src, output);
}
if (epilogue_params.Aux.data()) {
auto aux_output = output;
if constexpr (IsScalingAndAmaxAuxOutputNeeded) {
maximum_absolute_value_reduction<ElementCompute, true> amax_op;
local_abs_max_aux_output = amax_op(local_abs_max_aux_output, aux_output);
aux_output = epilogue_fma(converted_scale_aux, aux_output, ElementCompute(0));
if constexpr (IsBackpropFusion) {
ElementAux aux_input = ElementAux(0);
if (raw_pointer_cast(epilogue_params.Aux.data())) {
aux_input = epilogue_params.Aux(m + m_b, n + n_b, l);
}
epilogue_params.Aux(m + m_b, n + n_b, l) = aux_destination_converter(aux_output);
output = activation(output, aux_source_converter(aux_input));
local_dBias = add(local_dBias, output);
}
else {
if (raw_pointer_cast(epilogue_params.Aux.data())) {
auto aux_output = output;
if constexpr (IsScalingAndAmaxAuxOutputNeeded) {
maximum_absolute_value_reduction<ElementCompute, true> amax_op;
local_abs_max_aux_output = amax_op(local_abs_max_aux_output, aux_output);
aux_output = epilogue_fma(converted_scale_aux, aux_output, ElementCompute(0));
}
output = activation(output);
if constexpr (IsReLUAuxNeeded) {
epilogue_params.Aux(m + m_b, n + n_b, l) = not (aux_output < 0) ? uint1b_t(1) : uint1b_t(0);
} else {
epilogue_params.Aux(m + m_b, n + n_b, l) = aux_destination_converter(aux_output);
}
}
output = activation(output);
}
if constexpr (IsScalingAndAmaxOutputNeeded) {
maximum_absolute_value_reduction<ElementCompute, true> amax_op;
@@ -340,8 +380,16 @@ void gett_epilogue(
epilogue_params.D(m + m_b, n + n_b, l) = destination_converter(output);
}
} // n_b
if (m + m_b < cute::size<0>(epilogue_params.D.layout()) && n < cute::size<1>(epilogue_params.D.layout())) {
if (raw_pointer_cast(epilogue_params.Bias.data()) && IsBackpropFusion) {
ElementCompute converted_dBias = bias_converter(epilogue_params.Bias(m + m_b));
local_dBias = add(local_dBias, converted_dBias);
epilogue_params.Bias(m + m_b) = dBias_converter(local_dBias);
}
}
}
} // m_b
#if defined(_OPENMP)
#pragma omp critical(Abs_Max_Data_Update)
#endif

9
tools/util/include/cutlass/util/reference/host/tensor_fill.h
View File

@@ -566,7 +566,6 @@ struct RandomUniformFunc {
// Random values are cast to integer after scaling by a power of two to facilitate error
// testing
Element result;
if (int_scale >= 0) {
rnd = double(int64_t(rnd * double(1 << int_scale))) / double(1 << int_scale);
result = static_cast<Element>(Real(rnd));
@@ -1253,15 +1252,15 @@ void TensorFillRandom(
TensorFillRandomGaussian(
view,
seed,
static_cast<Real>(dist.gaussian.mean),
static_cast<Real>(dist.gaussian.stddev),
dist.gaussian.mean,
dist.gaussian.stddev,
dist.int_scale);
} else if (dist.kind == Distribution::Uniform) {
TensorFillRandomUniform(
view,
seed,
static_cast<Real>(dist.uniform.max),
static_cast<Real>(dist.uniform.min),
dist.uniform.max,
dist.uniform.min,
dist.int_scale);
}
}