leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

CUTLASS 3.7 (#2045)

Browse Source 
* CUTLASS 3.7

* clean up changelog

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
Yujia Zhai
2025-01-18 09:53:07 -05:00
committed by GitHub
co-authored by yuzhai Haicheng Wu
parent 902dff3663
commit b78588d163
2030 changed files with 8947 additions and 3475 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/CMakeLists.txt
+1 -1
View File
@@ -1,4 +1,4 @@
# Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
#
# Redistribution and use in source and binary forms, with or without
tools/library/CMakeLists.txt
+2 -2
View File
@@ -1,4 +1,4 @@
# Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
#
# Redistribution and use in source and binary forms, with or without
@@ -285,7 +285,7 @@ execute_process(
--kernel-filter-file "${KERNEL_FILTER_FILE}"
--selected-kernel-list "${CUTLASS_LIBRARY_GENERATED_KERNEL_LIST_FILE}"
--cuda-version "${CUTLASS_GENERATOR_CUDA_COMPILER_VERSION}"
--log-level DEBUG
--log-level INFO
--disable-cutlass-package-imports
RESULT_VARIABLE cutlass_lib_INSTANCE_GENERATION_RESULT
OUTPUT_VARIABLE cutlass_lib_INSTANCE_GENERATION_OUTPUT
tools/library/include/cutlass/library/arch_mappings.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/include/cutlass/library/descriptions.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/include/cutlass/library/handle.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/include/cutlass/library/library.h
+21 -3
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -117,8 +117,7 @@ public:
void const *arguments,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const = 0;
cudaStream_t stream = nullptr) const = 0;
};
@@ -173,6 +172,9 @@ struct GemmArguments {
/// Enumerant indicating whether alpha/beta point to host or device memory
ScalarPointerMode pointer_mode{};
/// Whether to use PDL when launching the kernel
bool use_pdl{false};
};
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -253,6 +255,7 @@ struct GemmArrayArguments {
void const *alpha{nullptr};
void const *beta{nullptr};
ScalarPointerMode pointer_mode{};
bool use_pdl{false};
};
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -307,6 +310,8 @@ struct GemmUniversalArguments {
int swizzle_size{1};
int device_index{0};
bool use_pdl{false};
};
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -354,6 +359,7 @@ struct GemmPlanarComplexArguments {
int64_t batch_stride_C_imag{0};
int64_t batch_stride_D_real{0};
int64_t batch_stride_D_imag{0};
bool use_pdl{false};
};
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -394,6 +400,7 @@ struct GemmPlanarComplexArrayArguments {
void const * alpha{nullptr};
void const * beta{nullptr};
ScalarPointerMode pointer_mode{};
bool use_pdl{false};
};
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -425,6 +432,7 @@ struct GemmGroupedArguments {
void const *alpha{nullptr};
void const *beta{nullptr};
ScalarPointerMode pointer_mode{};
bool use_pdl{false};
};
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -461,6 +469,7 @@ struct SparseGemmArguments {
void const *beta{nullptr}; /// pointer to beta scalar
ScalarPointerMode pointer_mode{}; /// enumerant indicating whether alpha/beta pointers are host
/// or device pointers.
bool use_pdl{false}; /// Whether to use PDL when launching the kernel
};
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -519,6 +528,7 @@ struct RankKArguments {
int64_t batch_stride_B{0};
int64_t batch_stride_C{0};
int64_t batch_stride_D{0};
bool use_pdl{false};
};
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -570,6 +580,7 @@ struct TrmmArguments {
int64_t batch_stride_A{0};
int64_t batch_stride_B{0};
int64_t batch_stride_D{0};
bool use_pdl{false};
};
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -628,6 +639,7 @@ struct SymmArguments {
int64_t batch_stride_B{0};
int64_t batch_stride_C{0};
int64_t batch_stride_D{0};
bool use_pdl{false};
};
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -744,6 +756,9 @@ struct ConvArguments {
/// Enumerant indicating whether alpha/beta point to host or device memory
ScalarPointerMode pointer_mode{};
/// Whether to use PDL when launching the kernel
bool use_pdl{false};
};
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -796,6 +811,9 @@ struct ReductionArguments {
/// Enumerant indicating whether alpha/beta point to host or device memory
ScalarPointerMode pointer_mode{};
/// Whether to use PDL when launching the kernel
bool use_pdl{false};
};
} // namespace library
tools/library/include/cutlass/library/manifest.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/include/cutlass/library/operation_table.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/include/cutlass/library/singleton.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/include/cutlass/library/types.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/include/cutlass/library/util.h
+8 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -177,6 +177,13 @@ char const *to_string(RasterOrder type, bool pretty = false);
template<>
RasterOrder from_string<RasterOrder>(std::string const &str);
/// Converts a bool to a string
char const *to_string(bool type, bool pretty = false);
/// Convers a bool from a string
template<>
bool from_string<bool>(std::string const &str);
/// Lexical cast from int64_t to string
std::string lexical_cast(int64_t int_value);
tools/library/src/conv2d_operation.h
+11 -13
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -238,6 +238,10 @@ protected:
operator_args.ref_C.reset(static_cast<ElementC *>(const_cast<void *>(arguments->C)));
operator_args.ref_D.reset(static_cast<ElementC *>(const_cast<void *>(arguments->D)));
if (arguments->use_pdl) {
return Status::kErrorNotSupported;
}
return Status::kSuccess;
}
@@ -326,12 +330,7 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
cudaStream_t stream = nullptr) const {
OperatorArguments args;
@@ -495,6 +494,10 @@ protected:
operator_args.ref_D.reset(static_cast<ElementC *>(const_cast<void *>(arguments->D)));
operator_args.ref_reordered_B.reset(static_cast<ElementC *>(const_cast<void *>(arguments->reordered_B)));
if (arguments->use_pdl) {
return Status::kErrorNotSupported;
}
return Status::kSuccess;
}
@@ -583,12 +586,7 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
cudaStream_t stream = nullptr) const {
OperatorArguments args;
tools/library/src/conv3d_operation.h
+6 -7
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -229,6 +229,10 @@ protected:
operator_args.ref_C.reset(static_cast<ElementC *>(const_cast<void *>(arguments->C)));
operator_args.ref_D.reset(static_cast<ElementC *>(const_cast<void *>(arguments->D)));
if (arguments->use_pdl) {
return Status::kErrorNotSupported;
}
return Status::kSuccess;
}
@@ -317,12 +321,7 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
cudaStream_t stream = nullptr) const {
OperatorArguments args;
tools/library/src/conv_operation_3x.hpp
+3 -4
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -334,8 +334,7 @@ public:
void const* arguments,
void* host_workspace,
void* device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const override
cudaStream_t stream = nullptr) const override
{
auto status = Status::kInvalid;
@@ -361,7 +360,7 @@ public:
}
auto* op = reinterpret_cast<Operator*>(host_workspace);
return op->run(out_args, device_workspace, stream, nullptr, launch_with_pdl);
return op->run(out_args, device_workspace, stream, nullptr, in_args_ptr->use_pdl);
}
private:
tools/library/src/gemm_operation.h
+28 -39
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -206,6 +206,10 @@ protected:
return Status::kErrorInvalidProblem;
}
if (arguments->use_pdl) {
return Status::kErrorNotSupported;
}
operator_args.ref_A.reset(static_cast<ElementA const *>(arguments->A));
operator_args.ref_B.reset(static_cast<ElementB const *>(arguments->B));
operator_args.ref_C.reset(static_cast<ElementC const *>(arguments->C));
@@ -296,12 +300,7 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
cudaStream_t stream = nullptr) const {
OperatorArguments args;
@@ -420,6 +419,10 @@ protected:
operator_args.ref_D.reset(static_cast<ElementD *>(arguments->D));
operator_args.ref_E.reset(static_cast<ElementE const *>(arguments->E));
if (arguments->use_pdl) {
return Status::kErrorNotSupported;
}
return Status::kSuccess;
}
@@ -505,13 +508,8 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
cudaStream_t stream = nullptr) const {
OperatorArguments args;
Status status = update_arguments_(
@@ -634,6 +632,10 @@ protected:
operator_args.batch_stride_C = arguments->batch_stride_C;
operator_args.batch_stride_D = arguments->batch_stride_D;
if (arguments->use_pdl) {
return Status::kErrorNotSupported;
}
return Status::kSuccess;
}
@@ -731,12 +733,7 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
cudaStream_t stream = nullptr) const {
OperatorArguments args;
@@ -945,13 +942,7 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
cudaStream_t stream = nullptr) const {
OperatorArguments args;
Status status = update_arguments_(
@@ -1064,6 +1055,10 @@ protected:
operator_args.ptr_N = arguments->N;
operator_args.ptr_K = arguments->K;
if (arguments->use_pdl) {
return Status::kErrorNotSupported;
}
return Status::kSuccess;
}
@@ -1153,12 +1148,7 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
cudaStream_t stream = nullptr) const {
OperatorArguments args;
@@ -1265,6 +1255,10 @@ protected:
op_args.ldc = arguments->ldc;
op_args.ldd = arguments->ldd;
if (arguments->use_pdl) {
return Status::kErrorNotSupported;
}
return Status::kSuccess;
}
@@ -1362,12 +1356,7 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
cudaStream_t stream = nullptr) const {
OperatorArguments args;
tools/library/src/gemm_operation_3x.hpp
+16 -16
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -248,8 +248,17 @@ protected:
arguments->ldc, arguments->batch_stride_C);
operator_args.epilogue.dD = operator_args.epilogue.dC;
/* Query device SM count to pass onto the kernel as an argument, where needed */
/* Query device SM count and max active clusters to pass onto the kernel as an argument, where needed */
operator_args.hw_info.sm_count = arguments->sm_count;
if constexpr (Operator::ArchTag::kMinComputeCapability == 90) {
dim3 cluster_dims(cute::size<0>(typename Operator::GemmKernel::ClusterShape{}),
cute::size<1>(typename Operator::GemmKernel::ClusterShape{}),
cute::size<2>(typename Operator::GemmKernel::ClusterShape{}));
uint32_t threads_per_block = Operator::GemmKernel::MaxThreadsPerBlock;
void const* kernel_ptr = (void*)(device_kernel<typename Operator::GemmKernel>);
operator_args.hw_info.max_active_clusters = cutlass::KernelHardwareInfo::query_device_max_active_clusters(
cluster_dims, threads_per_block, kernel_ptr);
}
if constexpr (!std::is_const_v<decltype(operator_args.scheduler.max_swizzle_size)>) {
operator_args.scheduler.max_swizzle_size = arguments->swizzle_size;
}
@@ -275,20 +284,11 @@ public:
/// Returns success if the operation can proceed
Status can_implement(
void const *configuration_ptr, void const *arguments_ptr) const override {
GemmUniversalConfiguration const *configuration =
static_cast<GemmUniversalConfiguration const *>(configuration_ptr);
[[maybe_unused]] void const *configuration_ptr, void const *arguments_ptr) const override {
GemmUniversalArguments const *arguments =
static_cast<GemmUniversalArguments const *>(arguments_ptr);
OperatorArguments args;
// can_implement rules may need access to problem shape
args.problem_shape = cute::make_shape(
configuration->problem_size.m(),
configuration->problem_size.n(),
configuration->problem_size.k(),
configuration->batch_count);
auto status = update_arguments_(args, arguments);
if (status != Status::kSuccess) {
return status;
@@ -332,8 +332,7 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const override {
cudaStream_t stream = nullptr) const override {
OperatorArguments args;
Status status = update_arguments_(args, static_cast<GemmUniversalArguments const *>(arguments_ptr));
@@ -343,7 +342,8 @@ public:
Operator *op = static_cast<Operator *>(host_workspace);
// We need to call initialize() since we have to rebuild TMA desc for every new set of args
status = op->run(args, device_workspace, stream, nullptr, launch_with_pdl);
status = op->run(args, device_workspace, stream, nullptr,
static_cast<GemmUniversalArguments const *>(arguments_ptr)->use_pdl);
return status;
}
};
tools/library/src/handle.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/library_internal.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/manifest.cpp
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/operation_table.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/rank_2k_operation.h
+6 -7
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -223,6 +223,10 @@ protected:
operator_args.batch_stride_C = arguments->batch_stride_C;
operator_args.batch_stride_D = arguments->batch_stride_D;
if (arguments->use_pdl) {
return Status::kErrorNotSupported;
}
return Status::kSuccess;
}
@@ -314,12 +318,7 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
cudaStream_t stream = nullptr) const {
OperatorArguments args;
tools/library/src/rank_k_operation.h
+6 -7
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -221,6 +221,10 @@ protected:
operator_args.batch_stride_C = arguments->batch_stride_C;
operator_args.batch_stride_D = arguments->batch_stride_D;
if (arguments->use_pdl) {
return Status::kErrorNotSupported;
}
return Status::kSuccess;
}
@@ -310,12 +314,7 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
cudaStream_t stream = nullptr) const {
OperatorArguments args;
tools/library/src/reduction/init_reduction_operations.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reduction/reduction_device.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reduction/reduction_operation.h
+7 -8
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -145,6 +145,10 @@ protected:
operator_args.source.reset(static_cast<ElementOutput *>(const_cast<void *>(arguments->source)));
operator_args.destination.reset(static_cast<ElementOutput *>(const_cast<void *>(arguments->destination)));
if (arguments->use_pdl) {
return Status::kErrorNotSupported;
}
return Status::kSuccess;
}
@@ -231,13 +235,8 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
cudaStream_t stream = nullptr) const {
OperatorArguments args;
Status status = update_arguments_(
tools/library/src/reference/conv2d.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reference/conv3d.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reference/conv_reference_operation.h
+2 -7
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -432,12 +432,7 @@ public:
void const *arguments,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
cudaStream_t stream = nullptr) const {
ConvArguments const &args = *static_cast<ConvArguments const *>(arguments);
tools/library/src/reference/gemm_e4m3a_e4m3out.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reference/gemm_e4m3a_e5m2out.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reference/gemm_e5m2a_e4m3out.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reference/gemm_e5m2a_e5m2out.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reference/gemm_fp32out.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reference/gemm_fp8in_bf16out.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reference/gemm_fp8in_fp16out.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reference/gemm_fp8in_fp32out.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reference/gemm_fp_mixed_input.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reference/gemm_fp_other.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reference/gemm_int4.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reference/gemm_int8_interleaved_32.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reference/gemm_int8_interleaved_64.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reference/gemm_int_mixed_input.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reference/gemm_reference_operation.h
+2 -7
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -192,12 +192,7 @@ public:
void const *arguments,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
cudaStream_t stream = nullptr) const {
GemmUniversalConfiguration const &config = *static_cast<GemmUniversalConfiguration const *>(host_workspace);
GemmUniversalArguments const &args = *static_cast<GemmUniversalArguments const *>(arguments);
tools/library/src/reference/gemm_s8_s8_s32.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reference/gemm_u8_u8_s32.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/reference/initialize_reference_operations.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/singleton.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/library/src/sparse_gemm_operation_3x.hpp
+4 -4
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -398,8 +398,7 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const override {
cudaStream_t stream = nullptr) const override {
OperatorArguments operator_args;
@@ -421,7 +420,8 @@ public:
Operator *op = static_cast<Operator *>(host_workspace);
// We need to call initialize() since we have to rebuild TMA desc for every new set of args
status = op->run(operator_args, device_op_workspace_ptr, stream, nullptr, launch_with_pdl);
status = op->run(operator_args, device_op_workspace_ptr, stream, nullptr,
static_cast<GemmUniversalArguments const *>(arguments_ptr)->use_pdl);
return status;
}
tools/library/src/symm_operation.h
+6 -8
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -221,6 +221,10 @@ protected:
operator_args.batch_stride_C = arguments->batch_stride_C;
operator_args.batch_stride_D = arguments->batch_stride_D;
if (arguments->use_pdl) {
return Status::kErrorNotSupported;
}
return Status::kSuccess;
}
@@ -312,13 +316,7 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
cudaStream_t stream = nullptr) const {
OperatorArguments args;
Status status = update_arguments_(
tools/library/src/trmm_operation.h
+6 -7
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -215,6 +215,10 @@ protected:
operator_args.ptr_D = arguments->D;
operator_args.batch_stride_D = arguments->batch_stride_D;
if (arguments->use_pdl) {
return Status::kErrorNotSupported;
}
return Status::kSuccess;
}
@@ -304,12 +308,7 @@ public:
void const *arguments_ptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr,
bool launch_with_pdl = false) const {
if (launch_with_pdl) {
return Status::kErrorNotSupported;
}
cudaStream_t stream = nullptr) const {
OperatorArguments args;
tools/library/src/util.cu
+48 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1052,6 +1052,53 @@ RasterOrder from_string<RasterOrder>(std::string const &str) {
return RasterOrder::kInvalid;
}
///////////////////////////////////////////////////////////////////////////////////////////////////
static struct {
char const *text;
char const *pretty;
char const *character;
bool enumerant;
}
Bool_enumerants[] = {
{"true", "<true>", "t", true},
{"false", "<false>", "f", false},
};
/// Converts a RasterOrder enumerant to a string
char const *to_string(bool type, bool pretty) {
for (auto const & possible : Bool_enumerants) {
if (type == possible.enumerant) {
if (pretty) {
return possible.pretty;
}
else {
return possible.text;
}
}
}
return pretty ? "Invalid" : "invalid";
}
/// Converts a RasterOrder enumerant from a string
template <>
bool from_string<bool>(std::string const &str) {
for (auto const & possible : Bool_enumerants) {
if ((str.compare(possible.text) == 0) ||
(str.compare(possible.pretty) == 0) ||
(str.compare(possible.character) == 0)) {
return possible.enumerant;
}
}
return false;
}
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Lexical cast a string to a byte array. Returns true if cast is successful or false if invalid.
tools/profiler/CMakeLists.txt
+1 -1
View File
@@ -1,4 +1,4 @@
# Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
#
# Redistribution and use in source and binary forms, with or without
tools/profiler/include/cutlass/profiler/conv2d_operation_profiler.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/include/cutlass/profiler/conv3d_operation_profiler.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/include/cutlass/profiler/cublas_helpers.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/include/cutlass/profiler/cudnn_helpers.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/include/cutlass/profiler/cutlass_profiler.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/include/cutlass/profiler/debug.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/include/cutlass/profiler/device_allocation.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/include/cutlass/profiler/device_context.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/include/cutlass/profiler/enumerated_types.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/include/cutlass/profiler/gemm_operation_profiler.h
+3 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -92,6 +92,8 @@ public:
std::vector<uint8_t> alpha_one;
std::vector<uint8_t> beta_zero;
bool use_pdl{false};
//
// Methods
//
tools/profiler/include/cutlass/profiler/gpu_timer.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/include/cutlass/profiler/operation_profiler.h
+16 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -239,6 +239,21 @@ protected:
void *host_workspace,
void *device_workspace);
/// Profiles the GPU kernel launched in `func` running simultaneously on all
/// requested devices.
Status profile_kernel_(
PerformanceResult &result,
Options const &options,
const std::function<Status(int, cudaStream_t, int)> &func,
const std::vector<cudaStream_t> &streams);
/// Profiles the GPU kernel launched in `func` on the `stream`
Status profile_kernel_(
PerformanceResult &result,
Options const &options,
const std::function<Status(cudaStream_t, int)> &func,
cudaStream_t stream = nullptr);
private:
/// finds string matches filter_string in operation_name
bool find_string_matches_(
tools/profiler/include/cutlass/profiler/options.h
+8 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -199,8 +199,15 @@ public:
int warmup_iterations{10};
/// Number of iterations to profile each kernel - if 0, kernels are launched up to the profiling duration
/// This will always override profiling-duration and min-iterations.
int iterations{100};
/// Time to spend profiling each kernel (ms)
int duration{10};
/// Minimum number of iterations to profile
int min_iterations{10};
/// Number of ms to sleep between profiling periods (ms)
int sleep_duration{50};
tools/profiler/include/cutlass/profiler/performance_report.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/include/cutlass/profiler/performance_result.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/include/cutlass/profiler/problem_space.h
+8 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -873,6 +873,13 @@ bool arg_as_int(
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
bool arg_as_bool(bool &bool_value, KernelArgument::Value const *value_ptr);
bool arg_as_bool(bool &bool_value,
char const *name,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
bool arg_as_NumericTypeID(library::NumericTypeID &numeric_type, KernelArgument::Value const *value_ptr);
tools/profiler/include/cutlass/profiler/rank_2k_operation_profiler.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/include/cutlass/profiler/rank_k_operation_profiler.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/include/cutlass/profiler/reduction_operation_profiler.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/include/cutlass/profiler/sparse_gemm_operation_profiler.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/include/cutlass/profiler/symm_operation_profiler.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/include/cutlass/profiler/trmm_operation_profiler.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/src/conv2d_operation_profiler.cu
+7 -87
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1281,8 +1281,6 @@ Status Conv2dOperationProfiler::profile_cutlass_(
void *host_workspace,
void *device_workspace) {
GpuTimer timer;
// initialize conv2d underlying operation to handle parallel reduction
library::Operation const* underlying_operation = operation;
@@ -1294,23 +1292,9 @@ Status Conv2dOperationProfiler::profile_cutlass_(
}
}
//
// Optional sleep to limit power consumption and thermals
//
sleep(options.profiling.sleep_duration);
//
// Warmup loop
//
Status status;
for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
auto func = [&](cudaStream_t, int iteration) {
// Setup rotating workspace
int workspace_idx = options.profiling.warmup_iterations + iteration;
int problem_idx = (workspace_idx % conv_workspace_.problem_count);
int problem_idx = iteration % conv_workspace_.problem_count;
conv_arguments->A = conv_workspace_.A->batch_data(problem_idx);
conv_arguments->B = conv_workspace_.B->batch_data(problem_idx);
@@ -1328,7 +1312,7 @@ Status Conv2dOperationProfiler::profile_cutlass_(
}
// Run underlying conv2d operation
status = underlying_operation->run(
Status status = underlying_operation->run(
arguments,
host_workspace,
device_workspace);
@@ -1345,74 +1329,10 @@ Status Conv2dOperationProfiler::profile_cutlass_(
if (status != Status::kSuccess) {
return status;
}
}
return status;
};
//
// Initialize GPU timer
//
timer.start();
//
// Profiling loop
//
int Iterations = options.profiling.iterations;
int iteration = 0;
for (; iteration < Iterations; ++iteration) {
// Setup rotating workspace
int problem_idx = (iteration % conv_workspace_.problem_count);
conv_arguments->A = conv_workspace_.A->batch_data(problem_idx);
conv_arguments->B = conv_workspace_.B->batch_data(problem_idx);
conv_arguments->C = conv_workspace_.C->batch_data(problem_idx);
conv_arguments->D = conv_workspace_.Computed->batch_data(problem_idx);
if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {
// update library::ConvArguments for parallel split-k reduction
conv_arguments->D = conv_workspace_.device_workspace.data();
/// initialize library::ReductionArguments
conv_workspace_.reduction_arguments.workspace = conv_workspace_.device_workspace.data();
conv_workspace_.reduction_arguments.source = conv_workspace_.C->batch_data(problem_idx);
conv_workspace_.reduction_arguments.destination = conv_workspace_.Computed->batch_data(problem_idx);
}
// Run underlying conv2d operation
status = underlying_operation->run(
arguments,
host_workspace,
device_workspace);
// Run parallel reduction kernel for parallel split_k_mode
if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {
status = reduction_op_->run(
&conv_workspace_.reduction_arguments,
conv_workspace_.reduction_host_workspace.data(),
nullptr);
}
if (status != Status::kSuccess) {
return status;
}
}
//
// Wait for completion
//
timer.stop_and_wait();
//
// Update performance result
//
result.runtime = timer.duration(iteration);
return status;
return profile_kernel_(result, options, func);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
tools/profiler/src/conv3d_operation_profiler.cu
+7 -73
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1148,8 +1148,6 @@ Status Conv3dOperationProfiler::profile_cutlass_(
void *host_workspace,
void *device_workspace) {
GpuTimer timer;
// initialize conv2d underlying operation to handle parallel reduction
library::Operation const* underlying_operation = operation;
@@ -1159,68 +1157,14 @@ Status Conv3dOperationProfiler::profile_cutlass_(
}
}
//
// Optional sleep to limit power consumption and thermals
//
sleep(options.profiling.sleep_duration);
//
// Warmup loop
//
Status status;
for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
auto func = [&](cudaStream_t, int iteration) {
// Setup rotating workspace
int workspace_idx = options.profiling.warmup_iterations + iteration;
int problem_idx = (workspace_idx % conv_workspace_.problem_count);
int problem_idx = iteration % conv_workspace_.problem_count;
set_cutlass_operator_arguments_(problem_idx);
// Run underlying conv2d operation
status = underlying_operation->run(
arguments,
host_workspace,
device_workspace);
// Run parallel reduction kernel for parallel split_k_mode
if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {
status = reduction_op_->run(
&conv_workspace_.reduction_arguments,
conv_workspace_.reduction_host_workspace.data(),
nullptr);
}
if (status != Status::kSuccess) {
return status;
}
}
//
// Initialize GPU timer
//
timer.start();
//
// Profiling loop
//
int Iterations = options.profiling.iterations;
int iteration = 0;
for (; iteration < Iterations; ++iteration) {
// Setup rotating workspace
int problem_idx = (iteration % conv_workspace_.problem_count);
set_cutlass_operator_arguments_(problem_idx);
// Run underlying conv2d operation
status = underlying_operation->run(
Status status = underlying_operation->run(
arguments,
host_workspace,
device_workspace);
@@ -1236,21 +1180,11 @@ Status Conv3dOperationProfiler::profile_cutlass_(
if (status != Status::kSuccess) {
return status;
}
}
//
// Wait for completion
//
return status;
};
timer.stop_and_wait();
//
// Update performance result
//
result.runtime = timer.duration(iteration);
return status;
return profile_kernel_(result, options, func);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
tools/profiler/src/cublas_helpers.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/src/cudnn_helpers.cpp
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/src/cutlass_profiler.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/src/device_allocation.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/src/device_context.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/src/enumerated_types.cpp
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/src/gemm_operation_profiler.cu
+56 -174
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -76,6 +76,7 @@ GemmOperationProfiler::GemmOperationProfiler(Options const &options):
{ArgumentTypeID::kInteger, {"split_k_slices", "split-k-slices"}, "Number of partitions of K dimension"},
{ArgumentTypeID::kInteger, {"batch_count", "batch-count"}, "Number of GEMMs computed in one batch"},
{ArgumentTypeID::kEnumerated, {"raster_order", "raster-order"}, "Raster order (heuristic, along_n, along_m)"},
{ArgumentTypeID::kInteger, {"use_pdl", "use-pdl"}, "Use PDL (true, false)"},
{ArgumentTypeID::kInteger, {"swizzle_size", "swizzle-size"}, "Size to swizzle"},
},
{ library::Provider::kCUBLAS}
@@ -171,6 +172,11 @@ Status GemmOperationProfiler::GemmProblem::parse(
this->k = 1024;
}
if (!arg_as_bool(this->use_pdl, "use_pdl", problem_space, problem)) {
// default value
this->use_pdl = false;
}
if (!arg_as_SplitKModeID(this->split_k_mode, "split_k_mode", problem_space, problem)) {
// default value
this->split_k_mode = library::SplitKMode::kSerial;
@@ -337,6 +343,7 @@ void GemmOperationProfiler::GemmProblem::initialize_result(
set_argument(result, "batch_count", problem_space, batch_count);
set_argument(result, "raster_order", problem_space, library::to_string(raster_order));
set_argument(result, "swizzle_size", problem_space, swizzle_size);
set_argument(result, "use_pdl", problem_space, library::to_string(use_pdl));
set_argument(result, "alpha", problem_space,
library::lexical_cast(alpha, operation_desc.element_epilogue));
@@ -388,6 +395,7 @@ Status GemmOperationProfiler::initialize_configuration(
gemm_workspace_[i].configuration.device_count = static_cast<int>(device_count);
gemm_workspace_[i].arguments.device_index = static_cast<int>(i);
gemm_workspace_[i].arguments.use_pdl = problem_.use_pdl;
if (problem_.mode == library::GemmUniversalMode::kBatched) {
gemm_workspace_[i].configuration.batch_count = problem_.batch_count;
@@ -396,6 +404,16 @@ Status GemmOperationProfiler::initialize_configuration(
gemm_workspace_[i].configuration.batch_count = problem_.split_k_slices;
}
gemm_workspace_[i].arguments.problem_size.m() = int(problem_.m);
gemm_workspace_[i].arguments.problem_size.n() = int(problem_.n);
gemm_workspace_[i].arguments.problem_size.k() = int(problem_.k);
if (problem_.mode == library::GemmUniversalMode::kBatched) {
gemm_workspace_[i].arguments.batch_count = problem_.batch_count;
}
else {
gemm_workspace_[i].arguments.batch_count = problem_.split_k_slices;
}
gemm_workspace_[i].arguments.A = nullptr;
gemm_workspace_[i].arguments.B = nullptr;
gemm_workspace_[i].arguments.C = nullptr;
@@ -406,7 +424,6 @@ Status GemmOperationProfiler::initialize_configuration(
gemm_workspace_[i].arguments.swizzle_size = problem_.swizzle_size;
gemm_workspace_[i].arguments.raster_order = problem_.raster_order;
initialize_result_(this->model_result_, options, operation_desc, problem_space);
if (const auto can_implement = operation->can_implement(&gemm_workspace_[i].configuration, &gemm_workspace_[i].arguments); can_implement != Status::kSuccess) {
return can_implement;
}
@@ -1145,18 +1162,6 @@ bool GemmOperationProfiler::verify_with_reference_(
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace {
extern "C" {
__global__ void delay(cuda::atomic<bool> const* release) {
while (release->load(cuda::memory_order_acquire) != true) {
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 700)
__nanosleep(100);
#endif
}
}
}
}
/// Measures performance results
bool GemmOperationProfiler::profile(
Options const &options,
@@ -1219,15 +1224,6 @@ Status GemmOperationProfiler::profile_cutlass_(
void *,
void *) {
cuda::atomic<bool> *release;
cudaHostAlloc(&release, sizeof(*release), cudaHostAllocPortable);
release->store(false, cuda::memory_order_release);
std::vector<GpuTimer> timer;
for (size_t i = 0; i < gemm_workspace_.size(); ++i) {
cudaSetDevice(options.device.device_id(i));
timer.emplace_back();
}
// initialize gemm underlying operation to handle parallel reduction
library::Operation const * underlying_operation = operation;
@@ -1237,172 +1233,58 @@ Status GemmOperationProfiler::profile_cutlass_(
}
}
//
// Optional sleep to limit power consumption and thermals
//
auto launch_gemm = [&](int dev_id, cudaStream_t stream, int iteration) {
int problem_idx = (iteration % gemm_workspace_[dev_id].problem_count) * problem_.batch_count;
sleep(options.profiling.sleep_duration);
gemm_workspace_[dev_id].arguments.A = gemm_workspace_[dev_id].A->batch_data(problem_idx);
gemm_workspace_[dev_id].arguments.B = gemm_workspace_[dev_id].B->batch_data(problem_idx);
gemm_workspace_[dev_id].arguments.C = gemm_workspace_[dev_id].C->batch_data(problem_idx);
gemm_workspace_[dev_id].arguments.D = gemm_workspace_[dev_id].Computed->batch_data(problem_idx);
//
// Warmup loop
//
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
gemm_workspace_[dev_id].arguments.D = gemm_workspace_[dev_id].device_workspace.data();
Status status;
gemm_workspace_[dev_id].reduction_arguments.workspace = gemm_workspace_[dev_id].device_workspace.data();
gemm_workspace_[dev_id].reduction_arguments.source = gemm_workspace_[dev_id].C->batch_data(problem_idx);
gemm_workspace_[dev_id].reduction_arguments.destination = gemm_workspace_[dev_id].Computed->batch_data(problem_idx);
}
std::vector<cudaGraph_t> graphs;
graphs.resize(gemm_workspace_.size());
std::vector<cudaGraphExec_t> graphExecs;
graphExecs.resize(gemm_workspace_.size());
// Execute the CUTLASS operation
Status status = underlying_operation->run(
&gemm_workspace_[dev_id].arguments,
gemm_workspace_[dev_id].host_workspace.data(),
gemm_workspace_[dev_id].device_workspace.data(),
stream);
for (size_t i = 0; i < gemm_workspace_.size(); ++i) {
cudaSetDevice(options.device.device_id(i));
cudaStreamBeginCapture(gemm_workspace_[i].stream, cudaStreamCaptureModeGlobal);
// Halt execution until all GPUs are ready to precede.
// It allows the CPU to trigger the GPUs all start at the same time.
delay<<<1, 1, 0, gemm_workspace_[i].stream>>>(release);
for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
int problem_idx = (iteration % gemm_workspace_[i].problem_count) * problem_.batch_count;
if (status != Status::kSuccess) {
return status;
}
gemm_workspace_[i].arguments.A = gemm_workspace_[i].A->batch_data(problem_idx);
gemm_workspace_[i].arguments.B = gemm_workspace_[i].B->batch_data(problem_idx);
gemm_workspace_[i].arguments.C = gemm_workspace_[i].C->batch_data(problem_idx);
gemm_workspace_[i].arguments.D = gemm_workspace_[i].Computed->batch_data(problem_idx);
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
gemm_workspace_[i].arguments.D = gemm_workspace_[i].device_workspace.data();
gemm_workspace_[i].reduction_arguments.workspace = gemm_workspace_[i].device_workspace.data();
gemm_workspace_[i].reduction_arguments.source = gemm_workspace_[i].C->batch_data(problem_idx);
gemm_workspace_[i].reduction_arguments.destination = gemm_workspace_[i].Computed->batch_data(problem_idx);
}
// Execute the CUTLASS operation
status = underlying_operation->run(
&gemm_workspace_[i].arguments,
gemm_workspace_[i].host_workspace.data(),
gemm_workspace_[i].device_workspace.data(),
gemm_workspace_[i].stream);
// Run parallel reduction kernel for parallel split_k_mode
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
status = reduction_op_->run(
&gemm_workspace_[dev_id].reduction_arguments,
gemm_workspace_[dev_id].reduction_host_workspace.data(),
nullptr,
gemm_workspace_[dev_id].stream);
if (status != Status::kSuccess) {
return status;
}
// Run parallel reduction kernel for parallel split_k_mode
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
status = reduction_op_->run(
&gemm_workspace_[i].reduction_arguments,
gemm_workspace_[i].reduction_host_workspace.data(),
nullptr,
gemm_workspace_[i].stream);
if (status != Status::kSuccess) {
return status;
}
}
}
return Status::kSuccess;
};
//
// Initialize GPU timer
//
timer[i].start(gemm_workspace_[i].stream, cudaEventRecordExternal);
//
// Profiling loop
//
int Iterations = options.profiling.iterations;
int iteration = 0;
for (; iteration < Iterations; ++iteration) {
// Iterate over copies of the problem in memory
int workspace_idx = options.profiling.warmup_iterations + iteration;
int problem_idx = (workspace_idx % gemm_workspace_[i].problem_count) * problem_.batch_count;
gemm_workspace_[i].arguments.A = gemm_workspace_[i].A->batch_data(problem_idx);
gemm_workspace_[i].arguments.B = gemm_workspace_[i].B->batch_data(problem_idx);
gemm_workspace_[i].arguments.C = gemm_workspace_[i].C->batch_data(problem_idx);
gemm_workspace_[i].arguments.D = gemm_workspace_[i].Computed->batch_data(problem_idx);
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
gemm_workspace_[i].arguments.D = gemm_workspace_[i].device_workspace.data();
gemm_workspace_[i].reduction_arguments.workspace = gemm_workspace_[i].device_workspace.data();
gemm_workspace_[i].reduction_arguments.source = gemm_workspace_[i].C->batch_data(problem_idx);
gemm_workspace_[i].reduction_arguments.destination = gemm_workspace_[i].Computed->batch_data(problem_idx);
}
status = underlying_operation->run(
&gemm_workspace_[i].arguments,
gemm_workspace_[i].host_workspace.data(),
gemm_workspace_[i].device_workspace.data(),
gemm_workspace_[i].stream);
if (status != Status::kSuccess) {
return status;
}
// Run parallel reduction kernel for parallel split_k_mode
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
status = reduction_op_->run(
&gemm_workspace_[i].reduction_arguments,
gemm_workspace_[i].reduction_host_workspace.data(),
nullptr,
gemm_workspace_[i].stream);
if (status != Status::kSuccess) {
return status;
}
}
}
timer[i].stop(gemm_workspace_[i].stream, cudaEventRecordExternal);
cudaStreamEndCapture(gemm_workspace_[i].stream, &graphs[i]);
cudaGraphInstantiate(&graphExecs[i], graphs[i], nullptr, nullptr, 0);
if (options.device.devices.size() == 1) {
auto func = [&](cudaStream_t stream, int iteration) { return launch_gemm(0, stream, iteration); };
return profile_kernel_(result, options, func, gemm_workspace_[0].stream);
}
for (size_t i = 0; i < gemm_workspace_.size(); ++i) {
cudaSetDevice(options.device.device_id(i));
cudaGraphLaunch(graphExecs[i], gemm_workspace_[i].stream);
std::vector<cudaStream_t> streams(gemm_workspace_.size());
for (size_t i = 0; i < streams.size(); i++) {
streams[i] = gemm_workspace_[i].stream;
}
//
// Wait for completion
//
release->store(true, cuda::memory_order_release);
for (size_t i = 0; i < gemm_workspace_.size(); ++i) {
cudaSetDevice(options.device.device_id(i));
cudaStreamSynchronize(gemm_workspace_[i].stream);
}
//
// Update performance result
//
result.runtime = 0;
for (size_t i = 0; i < gemm_workspace_.size(); ++i) {
cudaSetDevice(options.device.device_id(i));
result.runtime_vector[i] = timer[i].duration(options.profiling.iterations);
result.runtime += result.runtime_vector[i];
}
result.runtime /= static_cast<double>(gemm_workspace_.size());
cudaFreeHost(release);
for (size_t i = 0; i < gemm_workspace_.size(); ++i) {
cudaSetDevice(options.device.device_id(i));
cudaGraphExecDestroy(graphExecs[i]);
cudaGraphDestroy(graphs[i]);
}
for (size_t i = 0; i < gemm_workspace_.size(); ++i) {
cudaSetDevice(options.device.device_id(gemm_workspace_.size() - i - 1));
timer.pop_back();
}
return status;
return profile_kernel_(result, options, launch_gemm, streams);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
tools/profiler/src/gpu_timer.cpp
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/src/main.cpp
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/src/operation_profiler.cu
+212 -69
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -47,6 +47,8 @@
// sleep not supported
#endif
#include <cuda/atomic>
#include "cutlass/profiler/options.h"
#include "cutlass/profiler/operation_profiler.h"
#include "cutlass/profiler/gpu_timer.h"
@@ -55,9 +57,18 @@
///////////////////////////////////////////////////////////////////////////////////////////////////
#define CUDA_CHECK(call) \
do { \
cudaError_t err = call; \
if (err != cudaSuccess) { \
std::cerr << "CUDA error at " << __FILE__ << ":" << __LINE__ << " code=" << err << " \"" \
<< cudaGetErrorString(err) << "\"\n"; \
return Status::kErrorInternal; \
} \
} while (0)
namespace cutlass {
namespace profiler {
///////////////////////////////////////////////////////////////////////////////////////////////////
OperationProfiler::OperationProfiler(): kind_(library::OperationKind::kInvalid) { }
@@ -656,6 +667,203 @@ void OperationProfiler::save_workspace(
///////////////////////////////////////////////////////////////////////////////////////////////////
namespace {
extern "C" {
__global__ void delay(cuda::atomic<bool> const *release) {
while (release->load(cuda::memory_order_acquire) != true) {
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 700)
__nanosleep(100);
#endif
}
}
}
Status predict_iters(
int &iterations,
Options const &options,
const std::function<Status(cudaStream_t, int)> &func,
cudaStream_t stream) {
// always use profiling-iterations if requested
if (options.profiling.iterations != 0) {
iterations = options.profiling.iterations;
return Status::kSuccess;
}
// otherwise run for as many iterations as necessary to
// meet profiling-duration
constexpr int CALIBRATION_ITERS = 5;
GpuTimer timer;
timer.start(stream);
for (int i = 0; i < CALIBRATION_ITERS; i++) {
Status status = func(stream, i);
if (status != Status::kSuccess) {
return status;
}
}
timer.stop_and_wait(stream);
double est_iters = options.profiling.duration / std::max(timer.duration(CALIBRATION_ITERS), 1e-6);
constexpr uint64_t MAX_ITERS = 1'000'000;
iterations = std::min(static_cast<uint64_t>(std::ceil(est_iters)), static_cast<uint64_t>(MAX_ITERS));
iterations = std::max(options.profiling.min_iterations, iterations);
return Status::kSuccess;
};
} // namespace
/// This profiling method is designed to run a kernel on several GPUs to
/// measure interference (e.g. due to power throttling).
/// To encourage the kernels to start at the same time and minimize jitter,
/// a spinloop kernel blocks each stream while work is being enqueued, which is
/// later triggered from the host.
/// CUDA graphs allows you to record the launch of large numbers of kernels without
/// blocking and therefore avoids a deadlock which happens if you try to enqueue too
/// many kernels behind the spinloop kernel.
Status OperationProfiler::profile_kernel_(
PerformanceResult &result,
Options const &options,
const std::function<Status(int, cudaStream_t, int)> &func,
const std::vector<cudaStream_t> &streams) {
auto dev_count = streams.size();
cuda::atomic<bool> *release;
CUDA_CHECK(cudaHostAlloc(&release, sizeof(*release), cudaHostAllocPortable));
release->store(false, cuda::memory_order_release);
std::vector<GpuTimer> timer;
for (size_t i = 0; i < dev_count; ++i) {
CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
timer.emplace_back();
}
std::vector<cudaGraph_t> graphs;
graphs.resize(dev_count);
std::vector<cudaGraphExec_t> graphExecs;
graphExecs.resize(dev_count);
sleep(options.profiling.sleep_duration);
// predict time by running on device 0
int iterations;
CUDA_CHECK(cudaSetDevice(0));
Status status = predict_iters(
iterations,
options,
[&](cudaStream_t stream, int iter) { return func(0, stream, iter); },
streams[0]);
if (status != Status::kSuccess) {
return status;
}
for (size_t i = 0; i < dev_count; ++i) {
CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
CUDA_CHECK(cudaStreamBeginCapture(streams[i], cudaStreamCaptureModeGlobal));
// Halt execution until all GPUs are ready to precede.
// It allows the CPU to trigger the GPUs all start at the same time.
delay<<<1, 1, 0, streams[i]>>>(release);
for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
Status status = func(i, streams[i], iteration);
if (status != Status::kSuccess) {
return status;
}
}
timer[i].start(streams[i], cudaEventRecordExternal);
int iteration = 0;
for (; iteration < iterations; ++iteration) {
Status status = func(i, streams[i], iteration + options.profiling.warmup_iterations);
if (status != Status::kSuccess) {
return status;
}
}
timer[i].stop(streams[i], cudaEventRecordExternal);
CUDA_CHECK(cudaStreamEndCapture(streams[i], &graphs[i]));
CUDA_CHECK(cudaGraphInstantiate(&graphExecs[i], graphs[i], nullptr, nullptr, 0));
}
for (size_t i = 0; i < dev_count; ++i) {
CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
CUDA_CHECK(cudaGraphLaunch(graphExecs[i], streams[i]));
}
// release the enqueued kernels
release->store(true, cuda::memory_order_release);
for (size_t i = 0; i < dev_count; ++i) {
CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
CUDA_CHECK(cudaStreamSynchronize(streams[i]));
}
result.runtime = 0;
for (size_t i = 0; i < dev_count; ++i) {
CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
result.runtime_vector[i] = timer[i].duration(iterations);
result.runtime += result.runtime_vector[i];
}
result.runtime /= static_cast<double>(dev_count);
CUDA_CHECK(cudaFreeHost(release));
for (size_t i = 0; i < dev_count; ++i) {
CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
CUDA_CHECK(cudaGraphExecDestroy(graphExecs[i]));
CUDA_CHECK(cudaGraphDestroy(graphs[i]));
}
for (size_t i = 0; i < dev_count; ++i) {
CUDA_CHECK(cudaSetDevice(options.device.device_id(dev_count - i - 1)));
timer.pop_back();
}
return Status::kSuccess;
}
/// Method to profile GPU execution time of a kernel launched in func
Status OperationProfiler::profile_kernel_(
PerformanceResult &result,
Options const &options,
const std::function<Status(cudaStream_t, int)> &func,
cudaStream_t stream) {
GpuTimer timer;
// Optional sleep to limit power consumption and thermals
sleep(options.profiling.sleep_duration);
Status status = Status::kSuccess;
int iterations;
status = predict_iters(iterations, options, func, stream);
if (status != Status::kSuccess) {
return status;
}
for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
status = func(stream, iteration);
if (status != Status::kSuccess) {
return status;
}
}
timer.start(stream);
int iteration = 0;
for (; iteration < iterations; ++iteration) {
status = func(stream, iteration + options.profiling.warmup_iterations);
if (status != Status::kSuccess) {
result.status = status;
return status;
}
}
timer.stop_and_wait(stream);
result.runtime = timer.duration(iteration);
result.status = status;
return status;
}
/// Method to profile a CUTLASS Operation
Status OperationProfiler::profile_cutlass_(
PerformanceResult &result,
@@ -665,70 +873,8 @@ Status OperationProfiler::profile_cutlass_(
void *host_workspace,
void *device_workspace) {
GpuTimer timer;
//
// Optional sleep to limit power consumption and thermals
//
sleep(options.profiling.sleep_duration);
//
// Warmup loop
//
Status status;
for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
status = operation->run(
arguments,
host_workspace,
device_workspace);
if (status != Status::kSuccess) {
return status;
}
}
//
// Initialize GPU timer
//
timer.start();
//
// Profiling loop
//
int Iterations = options.profiling.iterations;
int iteration = 0;
for (; iteration < Iterations; ++iteration) {
status = operation->run(
arguments,
host_workspace,
device_workspace);
if (status != Status::kSuccess) {
return status;
}
}
//
// Wait for completion
//
timer.stop_and_wait();
//
// Update performance result
//
result.runtime = timer.duration(iteration);
return status;
auto op = [=](cudaStream_t, int) { return operation->run(arguments, host_workspace, device_workspace); };
return profile_kernel_(result, options, op);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
@@ -748,9 +894,6 @@ void OperationProfiler::initialize_result_(
set_argument(result, "cta_m", problem_space, operation_desc.tile_description.threadblock_shape.m());
set_argument(result, "cta_n", problem_space, operation_desc.tile_description.threadblock_shape.n());
set_argument(result, "cta_k", problem_space, operation_desc.tile_description.threadblock_shape.k());
set_argument(result, "cluster_m", problem_space, operation_desc.tile_description.cluster_shape.m());
set_argument(result, "cluster_n", problem_space, operation_desc.tile_description.cluster_shape.n());
set_argument(result, "cluster_k", problem_space, operation_desc.tile_description.cluster_shape.k());
set_argument(result, "stages", problem_space, operation_desc.tile_description.threadblock_stages);
set_argument(result, "warps_m", problem_space, operation_desc.tile_description.warp_count.m());
set_argument(result, "warps_n", problem_space, operation_desc.tile_description.warp_count.n());
tools/profiler/src/options.cu
+14 -2
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -474,6 +474,8 @@ Options::Profiling::Profiling(cutlass::CommandLine const &cmdline) {
cmdline.get_cmd_line_argument("profiling-iterations", iterations, 100);
cmdline.get_cmd_line_argument("sleep-duration", sleep_duration, 50);
cmdline.get_cmd_line_argument("profiling-enabled", enabled, true);
cmdline.get_cmd_line_argument("profiling-duration", duration, 10);
cmdline.get_cmd_line_argument("min-iterations", min_iterations, 10);
if (cmdline.check_cmd_line_flag("providers")) {
@@ -504,7 +506,17 @@ void Options::Profiling::print_usage(std::ostream &out) const {
<< " --profiling-iterations=<iterations> "
<< " Number of iterations to profile each kernel. If zero, kernels" << end_of_line
<< " are launched up to the profiling duration.\n\n"
<< " are launched up to the profiling duration. If non-zero, this overrides" << end_of_line
<< " --profiling-duration and --min-iterations.\n\n"
<< " --profiling-duration=<duration> "
<< " Time to spend profiling each kernel (ms)." << end_of_line
<< " Overriden by `profiling-iterations` when `profiling-iterations` > 0." << end_of_line
<< " Note that `min-iterations` must also be satisfied.\n\n"
<< " --min-iterations=<iterations> "
<< " Minimum number of iterations to spend profiling each kernel, even if" << end_of_line
<< " `profiling-duration` has been met.\n\n"
<< " --warmup-iterations=<iterations> "
<< " Number of iterations to execute each kernel prior to profiling.\n\n"
tools/profiler/src/performance_report.cpp
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/src/performance_result.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/profiler/src/problem_space.cpp
+39 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -800,6 +800,44 @@ bool arg_as_int(
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Lexically casts an argument to an bool if it is defined. Returns true if not null.
bool arg_as_bool(bool &bool_value, KernelArgument::Value const *value_ptr) {
if (value_ptr->not_null) {
if (value_ptr->argument->description->type == ArgumentTypeID::kInteger) {
int64_t value64;
arg_as_int(value64, value_ptr);
bool_value = static_cast<bool>(value64);
}
else if (value_ptr->argument->description->type == ArgumentTypeID::kEnumerated) {
bool_value = library::from_string<bool>(
static_cast<EnumeratedTypeArgument::EnumeratedTypeValue const *>(value_ptr)->element);
}
else {
throw std::runtime_error(
"arg_as_bool() - illegal cast. Problem space argument must be integer or enumerated");
}
return true;
}
return false;
}
/// Lexically casts an argument to a bool
bool arg_as_bool(
bool &bool_value,
char const *name,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
size_t idx = problem_space.argument_index(name);
KernelArgument::Value const *value_ptr = problem.at(idx).get();
return arg_as_bool(bool_value, value_ptr);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
bool arg_as_NumericTypeID(
library::NumericTypeID &numeric_type,
tools/profiler/src/rank_2k_operation_profiler.cu
+5 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -297,6 +297,10 @@ void Rank2KOperationProfiler::RankKProblem::initialize_result(
set_argument(result, "n", problem_space, n);
set_argument(result, "k", problem_space, k);
set_argument(result, "cluster_m", problem_space, operation_desc.tile_description.cluster_shape.m());
set_argument(result, "cluster_n", problem_space, operation_desc.tile_description.cluster_shape.n());
set_argument(result, "cluster_k", problem_space, operation_desc.tile_description.cluster_shape.k());
set_argument(result, "split_k_slices", problem_space, split_k_slices);
set_argument(result, "batch_count", problem_space, batch_count);
tools/profiler/src/rank_k_operation_profiler.cu
+5 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -286,6 +286,10 @@ void RankKOperationProfiler::RankKProblem::initialize_result(
set_argument(result, "n", problem_space, n);
set_argument(result, "k", problem_space, k);
set_argument(result, "cluster_m", problem_space, operation_desc.tile_description.cluster_shape.m());
set_argument(result, "cluster_n", problem_space, operation_desc.tile_description.cluster_shape.n());
set_argument(result, "cluster_k", problem_space, operation_desc.tile_description.cluster_shape.k());
set_argument(result, "split_k_slices", problem_space, split_k_slices);
set_argument(result, "batch_count", problem_space, batch_count);
tools/profiler/src/sparse_gemm_operation_profiler.cu
+5 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -240,6 +240,10 @@ void SparseGemmOperationProfiler::SparseGemmProblem::initialize_result(
set_argument(result, "n", problem_space, n);
set_argument(result, "k", problem_space, k);
set_argument(result, "cluster_m", problem_space, operation_desc.tile_description.cluster_shape.m());
set_argument(result, "cluster_n", problem_space, operation_desc.tile_description.cluster_shape.n());
set_argument(result, "cluster_k", problem_space, operation_desc.tile_description.cluster_shape.k());
set_argument(result, "split_k_slices", problem_space, split_k_slices);
set_argument(result, "batch_count", problem_space, batch_count);
tools/profiler/src/symm_operation_profiler.cu
+5 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -320,6 +320,10 @@ void SymmOperationProfiler::SymmProblem::initialize_result(
set_argument(result, "m", problem_space, m);
set_argument(result, "n", problem_space, n);
set_argument(result, "cluster_m", problem_space, operation_desc.tile_description.cluster_shape.m());
set_argument(result, "cluster_n", problem_space, operation_desc.tile_description.cluster_shape.n());
set_argument(result, "cluster_k", problem_space, operation_desc.tile_description.cluster_shape.k());
set_argument(result, "split_k_slices", problem_space, split_k_slices);
set_argument(result, "batch_count", problem_space, batch_count);
tools/profiler/src/trmm_operation_profiler.cu
+5 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -251,6 +251,10 @@ void TrmmOperationProfiler::TrmmProblem::initialize_result(
set_argument(result, "m", problem_space, m);
set_argument(result, "n", problem_space, n);
set_argument(result, "cluster_m", problem_space, operation_desc.tile_description.cluster_shape.m());
set_argument(result, "cluster_n", problem_space, operation_desc.tile_description.cluster_shape.n());
set_argument(result, "cluster_k", problem_space, operation_desc.tile_description.cluster_shape.k());
set_argument(result, "split_k_slices", problem_space, split_k_slices);
set_argument(result, "batch_count", problem_space, batch_count);
tools/util/CMakeLists.txt
+1 -1
View File
@@ -1,4 +1,4 @@
# Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
#
# Redistribution and use in source and binary forms, with or without
tools/util/include/cutlass/util/GPU_Clock.hpp
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/util/include/cutlass/util/command_line.h
+1 -1
View File
@@ -1,5 +1,5 @@
/******************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/util/include/cutlass/util/cublas_wrappers.hpp
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
tools/util/include/cutlass/util/debug.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without

Some files were not shown because too many files have changed in this diff Show More