CUTLASS 3.7 (#2045)

* CUTLASS 3.7

* clean up changelog

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>

tools/profiler/src/conv2d_operation_profiler.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without
@@ -1281,8 +1281,6 @@ Status Conv2dOperationProfiler::profile_cutlass_(
   void *host_workspace,
   void *device_workspace) {
 
-  GpuTimer timer;
-
   // initialize conv2d underlying operation to handle parallel reduction
   library::Operation const* underlying_operation = operation;
 
@@ -1294,23 +1292,9 @@ Status Conv2dOperationProfiler::profile_cutlass_(
     }
   }
 
-  //
-  // Optional sleep to limit power consumption and thermals
-  //
-
-  sleep(options.profiling.sleep_duration);
-
-  //
-  // Warmup loop
-  //
-
-  Status status;
-
-  for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
-
+  auto func = [&](cudaStream_t, int iteration) {
     // Setup rotating workspace
-    int workspace_idx = options.profiling.warmup_iterations + iteration;
-    int problem_idx = (workspace_idx % conv_workspace_.problem_count);
+    int problem_idx = iteration % conv_workspace_.problem_count;
 
     conv_arguments->A = conv_workspace_.A->batch_data(problem_idx);
     conv_arguments->B = conv_workspace_.B->batch_data(problem_idx);
@@ -1328,7 +1312,7 @@ Status Conv2dOperationProfiler::profile_cutlass_(
     }
 
     // Run underlying conv2d operation
-    status = underlying_operation->run(
+    Status status = underlying_operation->run(
       arguments,
       host_workspace,
       device_workspace);
@@ -1345,74 +1329,10 @@ Status Conv2dOperationProfiler::profile_cutlass_(
     if (status != Status::kSuccess) {
       return status;
     }
-  }
+    return status;
+  };
 
-  //
-  // Initialize GPU timer
-  //
-
-  timer.start();
-
-  //
-  // Profiling loop
-  //
-
-  int Iterations = options.profiling.iterations;
-
-  int iteration = 0;
-  for (; iteration < Iterations; ++iteration) {
-
-    // Setup rotating workspace
-    int problem_idx = (iteration % conv_workspace_.problem_count);
-
-    conv_arguments->A = conv_workspace_.A->batch_data(problem_idx);
-    conv_arguments->B = conv_workspace_.B->batch_data(problem_idx);
-    conv_arguments->C = conv_workspace_.C->batch_data(problem_idx);
-    conv_arguments->D = conv_workspace_.Computed->batch_data(problem_idx);
-
-    if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {
-      // update library::ConvArguments for parallel split-k reduction
-      conv_arguments->D = conv_workspace_.device_workspace.data();
-
-      /// initialize library::ReductionArguments
-      conv_workspace_.reduction_arguments.workspace           = conv_workspace_.device_workspace.data();
-      conv_workspace_.reduction_arguments.source              = conv_workspace_.C->batch_data(problem_idx);
-      conv_workspace_.reduction_arguments.destination         = conv_workspace_.Computed->batch_data(problem_idx);
-    }
-
-    // Run underlying conv2d operation
-    status = underlying_operation->run(
-      arguments,
-      host_workspace,
-      device_workspace);
-
-    // Run parallel reduction kernel for parallel split_k_mode
-    if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {
-
-      status = reduction_op_->run(
-        &conv_workspace_.reduction_arguments,
-        conv_workspace_.reduction_host_workspace.data(),
-        nullptr);
-    }
-
-    if (status != Status::kSuccess) {
-      return status;
-    }
-  }
-
-  //
-  // Wait for completion
-  //
-
-  timer.stop_and_wait();
-
-  //
-  // Update performance result
-  //
-
-  result.runtime = timer.duration(iteration);
-
-  return status;
+  return profile_kernel_(result, options, func);
 }
 
 /////////////////////////////////////////////////////////////////////////////////////////////////

tools/profiler/src/conv3d_operation_profiler.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without
@@ -1148,8 +1148,6 @@ Status Conv3dOperationProfiler::profile_cutlass_(
   void *host_workspace,
   void *device_workspace) {
 
-  GpuTimer timer;
-
   // initialize conv2d underlying operation to handle parallel reduction
   library::Operation const* underlying_operation = operation;
 
@@ -1159,68 +1157,14 @@ Status Conv3dOperationProfiler::profile_cutlass_(
     }
   }
 
-  //
-  // Optional sleep to limit power consumption and thermals
-  //
-
-  sleep(options.profiling.sleep_duration);
-
-  //
-  // Warmup loop
-  //
-
-  Status status;
-
-  for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
-
+  auto func = [&](cudaStream_t, int iteration) {
     // Setup rotating workspace
-    int workspace_idx = options.profiling.warmup_iterations + iteration;
-    int problem_idx = (workspace_idx % conv_workspace_.problem_count);
+    int problem_idx = iteration % conv_workspace_.problem_count;
 
     set_cutlass_operator_arguments_(problem_idx);
 
     // Run underlying conv2d operation
-    status = underlying_operation->run(
-      arguments,
-      host_workspace,
-      device_workspace);
-
-    // Run parallel reduction kernel for parallel split_k_mode
-    if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {
-
-      status = reduction_op_->run(
-        &conv_workspace_.reduction_arguments,
-        conv_workspace_.reduction_host_workspace.data(),
-        nullptr);
-    }
-
-    if (status != Status::kSuccess) {
-      return status;
-    }
-  }
-
-  //
-  // Initialize GPU timer
-  //
-
-  timer.start();
-
-  //
-  // Profiling loop
-  //
-
-  int Iterations = options.profiling.iterations;
-
-  int iteration = 0;
-  for (; iteration < Iterations; ++iteration) {
-
-    // Setup rotating workspace
-    int problem_idx = (iteration % conv_workspace_.problem_count);
-
-    set_cutlass_operator_arguments_(problem_idx);
-
-    // Run underlying conv2d operation
-    status = underlying_operation->run(
+    Status status = underlying_operation->run(
       arguments,
       host_workspace,
       device_workspace);
@@ -1236,21 +1180,11 @@ Status Conv3dOperationProfiler::profile_cutlass_(
     if (status != Status::kSuccess) {
       return status;
     }
-  }
 
-  //
-  // Wait for completion
-  //
+    return status;
+  };
 
-  timer.stop_and_wait();
-
-  //
-  // Update performance result
-  //
-
-  result.runtime = timer.duration(iteration);
-
-  return status;
+  return profile_kernel_(result, options, func);
 }
 
 /////////////////////////////////////////////////////////////////////////////////////////////////

tools/profiler/src/cublas_helpers.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without

tools/profiler/src/cudnn_helpers.cpp

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without

tools/profiler/src/cutlass_profiler.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without

tools/profiler/src/device_allocation.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without

tools/profiler/src/device_context.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without

tools/profiler/src/enumerated_types.cpp

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without

tools/profiler/src/gemm_operation_profiler.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without
@@ -76,6 +76,7 @@ GemmOperationProfiler::GemmOperationProfiler(Options const &options):
       {ArgumentTypeID::kInteger, {"split_k_slices", "split-k-slices"}, "Number of partitions of K dimension"},
       {ArgumentTypeID::kInteger, {"batch_count", "batch-count"}, "Number of GEMMs computed in one batch"},
       {ArgumentTypeID::kEnumerated, {"raster_order", "raster-order"}, "Raster order (heuristic, along_n, along_m)"},
+      {ArgumentTypeID::kInteger, {"use_pdl", "use-pdl"}, "Use PDL (true, false)"}, 
       {ArgumentTypeID::kInteger, {"swizzle_size", "swizzle-size"}, "Size to swizzle"},
     },
     { library::Provider::kCUBLAS}
@@ -171,6 +172,11 @@ Status GemmOperationProfiler::GemmProblem::parse(
     this->k = 1024;
   }
 
+  if (!arg_as_bool(this->use_pdl, "use_pdl", problem_space, problem)) {
+    // default value
+    this->use_pdl = false;
+  }
+
   if (!arg_as_SplitKModeID(this->split_k_mode, "split_k_mode", problem_space, problem)) {
     // default value
     this->split_k_mode = library::SplitKMode::kSerial;
@@ -337,6 +343,7 @@ void GemmOperationProfiler::GemmProblem::initialize_result(
   set_argument(result, "batch_count", problem_space, batch_count);
   set_argument(result, "raster_order", problem_space, library::to_string(raster_order));
   set_argument(result, "swizzle_size", problem_space, swizzle_size);
+  set_argument(result, "use_pdl", problem_space, library::to_string(use_pdl));
 
   set_argument(result, "alpha", problem_space,
     library::lexical_cast(alpha, operation_desc.element_epilogue));
@@ -388,6 +395,7 @@ Status GemmOperationProfiler::initialize_configuration(
 
     gemm_workspace_[i].configuration.device_count = static_cast<int>(device_count);
     gemm_workspace_[i].arguments.device_index = static_cast<int>(i);
+    gemm_workspace_[i].arguments.use_pdl = problem_.use_pdl;
 
     if (problem_.mode == library::GemmUniversalMode::kBatched) {
       gemm_workspace_[i].configuration.batch_count = problem_.batch_count;
@@ -396,6 +404,16 @@ Status GemmOperationProfiler::initialize_configuration(
       gemm_workspace_[i].configuration.batch_count = problem_.split_k_slices;
     }
 
+    gemm_workspace_[i].arguments.problem_size.m() = int(problem_.m);
+    gemm_workspace_[i].arguments.problem_size.n() = int(problem_.n);
+    gemm_workspace_[i].arguments.problem_size.k() = int(problem_.k);
+    if (problem_.mode == library::GemmUniversalMode::kBatched) {
+      gemm_workspace_[i].arguments.batch_count = problem_.batch_count;
+    }
+    else {
+      gemm_workspace_[i].arguments.batch_count = problem_.split_k_slices;
+    }
+
     gemm_workspace_[i].arguments.A = nullptr;
     gemm_workspace_[i].arguments.B = nullptr;
     gemm_workspace_[i].arguments.C = nullptr;
@@ -406,7 +424,6 @@ Status GemmOperationProfiler::initialize_configuration(
     gemm_workspace_[i].arguments.swizzle_size = problem_.swizzle_size;
     gemm_workspace_[i].arguments.raster_order = problem_.raster_order;
     initialize_result_(this->model_result_, options, operation_desc, problem_space);
-
     if (const auto can_implement = operation->can_implement(&gemm_workspace_[i].configuration, &gemm_workspace_[i].arguments); can_implement != Status::kSuccess) {
       return can_implement;
     }
@@ -1145,18 +1162,6 @@ bool GemmOperationProfiler::verify_with_reference_(
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
 
-namespace {
-extern "C" {
-  __global__ void delay(cuda::atomic<bool> const* release) {
-    while (release->load(cuda::memory_order_acquire) != true) {
-#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 700)
-      __nanosleep(100);
-#endif
-    }
-  }
-}
-}
-
 /// Measures performance results
 bool GemmOperationProfiler::profile(
   Options const &options,
@@ -1219,15 +1224,6 @@ Status GemmOperationProfiler::profile_cutlass_(
   void *,
   void *) {
 
-  cuda::atomic<bool> *release;
-  cudaHostAlloc(&release, sizeof(*release), cudaHostAllocPortable);
-  release->store(false, cuda::memory_order_release);
-
-  std::vector<GpuTimer> timer;
-  for (size_t i = 0; i < gemm_workspace_.size(); ++i) {
-    cudaSetDevice(options.device.device_id(i));
-    timer.emplace_back();
-  }
   // initialize gemm underlying operation to handle parallel reduction
   library::Operation const * underlying_operation = operation;
 
@@ -1237,172 +1233,58 @@ Status GemmOperationProfiler::profile_cutlass_(
     }
   }
 
-  //
-  // Optional sleep to limit power consumption and thermals
-  //
+  auto launch_gemm = [&](int dev_id, cudaStream_t stream, int iteration) {
+    int problem_idx = (iteration % gemm_workspace_[dev_id].problem_count) * problem_.batch_count;
 
-  sleep(options.profiling.sleep_duration);
+    gemm_workspace_[dev_id].arguments.A = gemm_workspace_[dev_id].A->batch_data(problem_idx);
+    gemm_workspace_[dev_id].arguments.B = gemm_workspace_[dev_id].B->batch_data(problem_idx);
+    gemm_workspace_[dev_id].arguments.C = gemm_workspace_[dev_id].C->batch_data(problem_idx);
+    gemm_workspace_[dev_id].arguments.D = gemm_workspace_[dev_id].Computed->batch_data(problem_idx);
 
-  //
-  // Warmup loop
-  //
+    if (problem_.split_k_mode == library::SplitKMode::kParallel) {
+      gemm_workspace_[dev_id].arguments.D                     = gemm_workspace_[dev_id].device_workspace.data();
 
-  Status status;
+      gemm_workspace_[dev_id].reduction_arguments.workspace   = gemm_workspace_[dev_id].device_workspace.data();
+      gemm_workspace_[dev_id].reduction_arguments.source      = gemm_workspace_[dev_id].C->batch_data(problem_idx);
+      gemm_workspace_[dev_id].reduction_arguments.destination = gemm_workspace_[dev_id].Computed->batch_data(problem_idx);
+    }
 
-  std::vector<cudaGraph_t> graphs;
-  graphs.resize(gemm_workspace_.size());
-  std::vector<cudaGraphExec_t> graphExecs;
-  graphExecs.resize(gemm_workspace_.size());
+    // Execute the CUTLASS operation
+    Status status = underlying_operation->run(
+      &gemm_workspace_[dev_id].arguments,
+      gemm_workspace_[dev_id].host_workspace.data(),
+      gemm_workspace_[dev_id].device_workspace.data(),
+      stream);
 
-  for (size_t i = 0; i < gemm_workspace_.size(); ++i) {
-    cudaSetDevice(options.device.device_id(i));
-    cudaStreamBeginCapture(gemm_workspace_[i].stream, cudaStreamCaptureModeGlobal);
-    // Halt execution until all GPUs are ready to precede.
-    // It allows the CPU to trigger the GPUs all start at the same time.
-    delay<<<1, 1, 0, gemm_workspace_[i].stream>>>(release);
-    for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
-      int problem_idx = (iteration % gemm_workspace_[i].problem_count) * problem_.batch_count;
+    if (status != Status::kSuccess) {
+      return status;
+    }
 
-      gemm_workspace_[i].arguments.A = gemm_workspace_[i].A->batch_data(problem_idx);
-      gemm_workspace_[i].arguments.B = gemm_workspace_[i].B->batch_data(problem_idx);
-      gemm_workspace_[i].arguments.C = gemm_workspace_[i].C->batch_data(problem_idx);
-      gemm_workspace_[i].arguments.D = gemm_workspace_[i].Computed->batch_data(problem_idx);
-
-      if (problem_.split_k_mode == library::SplitKMode::kParallel) {
-        gemm_workspace_[i].arguments.D                     = gemm_workspace_[i].device_workspace.data();
-
-        gemm_workspace_[i].reduction_arguments.workspace   = gemm_workspace_[i].device_workspace.data();
-        gemm_workspace_[i].reduction_arguments.source      = gemm_workspace_[i].C->batch_data(problem_idx);
-        gemm_workspace_[i].reduction_arguments.destination = gemm_workspace_[i].Computed->batch_data(problem_idx);
-      }
-
-      // Execute the CUTLASS operation
-      status = underlying_operation->run(
-        &gemm_workspace_[i].arguments,
-        gemm_workspace_[i].host_workspace.data(),
-        gemm_workspace_[i].device_workspace.data(),
-        gemm_workspace_[i].stream);
+    // Run parallel reduction kernel for parallel split_k_mode
+    if (problem_.split_k_mode == library::SplitKMode::kParallel) {
+      status = reduction_op_->run(
+        &gemm_workspace_[dev_id].reduction_arguments,
+        gemm_workspace_[dev_id].reduction_host_workspace.data(),
+        nullptr,
+        gemm_workspace_[dev_id].stream);
 
       if (status != Status::kSuccess) {
         return status;
       }
-
-      // Run parallel reduction kernel for parallel split_k_mode
-      if (problem_.split_k_mode == library::SplitKMode::kParallel) {
-        status = reduction_op_->run(
-          &gemm_workspace_[i].reduction_arguments,
-          gemm_workspace_[i].reduction_host_workspace.data(),
-          nullptr,
-          gemm_workspace_[i].stream);
-
-        if (status != Status::kSuccess) {
-          return status;
-        }
-      }
     }
+    return Status::kSuccess;
+  };
 
-    //
-    // Initialize GPU timer
-    //
-
-    timer[i].start(gemm_workspace_[i].stream, cudaEventRecordExternal);
-
-    //
-    // Profiling loop
-    //
-
-    int Iterations = options.profiling.iterations;
-
-    int iteration = 0;
-
-    for (; iteration < Iterations; ++iteration) {
-      // Iterate over copies of the problem in memory
-      int workspace_idx = options.profiling.warmup_iterations + iteration;
-      int problem_idx = (workspace_idx % gemm_workspace_[i].problem_count) * problem_.batch_count;
-
-      gemm_workspace_[i].arguments.A = gemm_workspace_[i].A->batch_data(problem_idx);
-      gemm_workspace_[i].arguments.B = gemm_workspace_[i].B->batch_data(problem_idx);
-      gemm_workspace_[i].arguments.C = gemm_workspace_[i].C->batch_data(problem_idx);
-      gemm_workspace_[i].arguments.D = gemm_workspace_[i].Computed->batch_data(problem_idx);
-
-      if (problem_.split_k_mode == library::SplitKMode::kParallel) {
-        gemm_workspace_[i].arguments.D                     = gemm_workspace_[i].device_workspace.data();
-
-        gemm_workspace_[i].reduction_arguments.workspace   = gemm_workspace_[i].device_workspace.data();
-        gemm_workspace_[i].reduction_arguments.source      = gemm_workspace_[i].C->batch_data(problem_idx);
-        gemm_workspace_[i].reduction_arguments.destination = gemm_workspace_[i].Computed->batch_data(problem_idx);
-      }
-
-      status = underlying_operation->run(
-        &gemm_workspace_[i].arguments,
-        gemm_workspace_[i].host_workspace.data(),
-        gemm_workspace_[i].device_workspace.data(),
-        gemm_workspace_[i].stream);
-
-      if (status != Status::kSuccess) {
-        return status;
-      }
-
-      // Run parallel reduction kernel for parallel split_k_mode
-      if (problem_.split_k_mode == library::SplitKMode::kParallel) {
-        status = reduction_op_->run(
-          &gemm_workspace_[i].reduction_arguments,
-          gemm_workspace_[i].reduction_host_workspace.data(),
-          nullptr,
-          gemm_workspace_[i].stream);
-
-        if (status != Status::kSuccess) {
-          return status;
-        }
-      }
-    }
-    timer[i].stop(gemm_workspace_[i].stream, cudaEventRecordExternal);
-    cudaStreamEndCapture(gemm_workspace_[i].stream, &graphs[i]);
-    cudaGraphInstantiate(&graphExecs[i], graphs[i], nullptr, nullptr, 0);
+  if (options.device.devices.size() == 1) {
+    auto func = [&](cudaStream_t stream, int iteration) { return launch_gemm(0, stream, iteration); };
+    return profile_kernel_(result, options, func, gemm_workspace_[0].stream);
   }
 
-  for (size_t i = 0; i < gemm_workspace_.size(); ++i) {
-    cudaSetDevice(options.device.device_id(i));
-    cudaGraphLaunch(graphExecs[i], gemm_workspace_[i].stream);
+  std::vector<cudaStream_t> streams(gemm_workspace_.size());
+  for (size_t i = 0; i < streams.size(); i++) {
+    streams[i] = gemm_workspace_[i].stream;
   }
-
-  //
-  // Wait for completion
-  //
-
-  release->store(true, cuda::memory_order_release);
-
-  for (size_t i = 0; i < gemm_workspace_.size(); ++i) {
-    cudaSetDevice(options.device.device_id(i));
-    cudaStreamSynchronize(gemm_workspace_[i].stream);
-  }
-  //
-  // Update performance result
-  //
-
-
-  result.runtime = 0;
-  for (size_t i = 0; i < gemm_workspace_.size(); ++i) {
-    cudaSetDevice(options.device.device_id(i));
-    result.runtime_vector[i] = timer[i].duration(options.profiling.iterations);
-    result.runtime += result.runtime_vector[i];
-  }
-  result.runtime /= static_cast<double>(gemm_workspace_.size());
-
-  cudaFreeHost(release);
-
-  for (size_t i = 0; i < gemm_workspace_.size(); ++i) {
-    cudaSetDevice(options.device.device_id(i));
-    cudaGraphExecDestroy(graphExecs[i]);
-    cudaGraphDestroy(graphs[i]);
-  }
-
-  for (size_t i = 0; i < gemm_workspace_.size(); ++i) {
-    cudaSetDevice(options.device.device_id(gemm_workspace_.size() - i - 1));
-    timer.pop_back();
-  }
-
-  return status;
+  return profile_kernel_(result, options, launch_gemm, streams);
 }
 
 /////////////////////////////////////////////////////////////////////////////////////////////////

tools/profiler/src/gpu_timer.cpp

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without

tools/profiler/src/main.cpp

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without

tools/profiler/src/operation_profiler.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without
@@ -47,6 +47,8 @@
 // sleep not supported
 #endif
 
+#include <cuda/atomic>
+
 #include "cutlass/profiler/options.h"
 #include "cutlass/profiler/operation_profiler.h"
 #include "cutlass/profiler/gpu_timer.h"
@@ -55,9 +57,18 @@
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 
+#define CUDA_CHECK(call)                                                                                               \
+  do {                                                                                                                 \
+    cudaError_t err = call;                                                                                            \
+    if (err != cudaSuccess) {                                                                                          \
+      std::cerr << "CUDA error at " << __FILE__ << ":" << __LINE__ << " code=" << err << " \""                         \
+                << cudaGetErrorString(err) << "\"\n";                                                                  \
+      return Status::kErrorInternal;                                                                                   \
+    }                                                                                                                  \
+  } while (0)
+
 namespace cutlass {
 namespace profiler {
-
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 
 OperationProfiler::OperationProfiler(): kind_(library::OperationKind::kInvalid) { }
@@ -656,6 +667,203 @@ void OperationProfiler::save_workspace(
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 
+namespace {
+extern "C" {
+__global__ void delay(cuda::atomic<bool> const *release) {
+  while (release->load(cuda::memory_order_acquire) != true) {
+#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 700)
+    __nanosleep(100);
+#endif
+  }
+}
+}
+
+Status predict_iters(
+  int &iterations,
+  Options const &options,
+  const std::function<Status(cudaStream_t, int)> &func,
+  cudaStream_t stream) {
+  // always use profiling-iterations if requested
+  if (options.profiling.iterations != 0) {
+    iterations = options.profiling.iterations;
+    return Status::kSuccess;
+  }
+
+  // otherwise run for as many iterations as necessary to
+  // meet profiling-duration
+  constexpr int CALIBRATION_ITERS = 5;
+  GpuTimer timer;
+  timer.start(stream);
+  for (int i = 0; i < CALIBRATION_ITERS; i++) {
+    Status status = func(stream, i);
+    if (status != Status::kSuccess) {
+      return status;
+    }
+  }
+  timer.stop_and_wait(stream);
+
+  double est_iters             = options.profiling.duration / std::max(timer.duration(CALIBRATION_ITERS), 1e-6);
+  constexpr uint64_t MAX_ITERS = 1'000'000;
+  iterations = std::min(static_cast<uint64_t>(std::ceil(est_iters)), static_cast<uint64_t>(MAX_ITERS));
+  iterations = std::max(options.profiling.min_iterations, iterations);
+  return Status::kSuccess;
+};
+
+} // namespace
+
+/// This profiling method is designed to run a kernel on several GPUs to
+/// measure interference (e.g. due to power throttling).
+/// To encourage the kernels to start at the same time and minimize jitter,
+/// a spinloop kernel blocks each stream while work is being enqueued, which is
+/// later triggered from the host.
+/// CUDA graphs allows you to record the launch of large numbers of kernels without
+/// blocking and therefore avoids a deadlock which happens if you try to enqueue too
+/// many kernels behind the spinloop kernel.
+Status OperationProfiler::profile_kernel_(
+  PerformanceResult &result,
+  Options const &options,
+  const std::function<Status(int, cudaStream_t, int)> &func,
+  const std::vector<cudaStream_t> &streams) {
+  auto dev_count = streams.size();
+  cuda::atomic<bool> *release;
+  CUDA_CHECK(cudaHostAlloc(&release, sizeof(*release), cudaHostAllocPortable));
+  release->store(false, cuda::memory_order_release);
+
+  std::vector<GpuTimer> timer;
+  for (size_t i = 0; i < dev_count; ++i) {
+    CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
+    timer.emplace_back();
+  }
+
+  std::vector<cudaGraph_t> graphs;
+  graphs.resize(dev_count);
+  std::vector<cudaGraphExec_t> graphExecs;
+  graphExecs.resize(dev_count);
+
+  sleep(options.profiling.sleep_duration);
+
+  // predict time by running on device 0
+  int iterations;
+  CUDA_CHECK(cudaSetDevice(0));
+  Status status = predict_iters(
+    iterations,
+    options,
+    [&](cudaStream_t stream, int iter) { return func(0, stream, iter); },
+    streams[0]);
+  if (status != Status::kSuccess) {
+    return status;
+  }
+
+  for (size_t i = 0; i < dev_count; ++i) {
+    CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
+    CUDA_CHECK(cudaStreamBeginCapture(streams[i], cudaStreamCaptureModeGlobal));
+    // Halt execution until all GPUs are ready to precede.
+    // It allows the CPU to trigger the GPUs all start at the same time.
+    delay<<<1, 1, 0, streams[i]>>>(release);
+    for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
+      Status status = func(i, streams[i], iteration);
+      if (status != Status::kSuccess) {
+        return status;
+      }
+    }
+
+    timer[i].start(streams[i], cudaEventRecordExternal);
+
+    int iteration = 0;
+    for (; iteration < iterations; ++iteration) {
+      Status status = func(i, streams[i], iteration + options.profiling.warmup_iterations);
+      if (status != Status::kSuccess) {
+        return status;
+      }
+    }
+    timer[i].stop(streams[i], cudaEventRecordExternal);
+    CUDA_CHECK(cudaStreamEndCapture(streams[i], &graphs[i]));
+    CUDA_CHECK(cudaGraphInstantiate(&graphExecs[i], graphs[i], nullptr, nullptr, 0));
+  }
+
+  for (size_t i = 0; i < dev_count; ++i) {
+    CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
+    CUDA_CHECK(cudaGraphLaunch(graphExecs[i], streams[i]));
+  }
+
+  // release the enqueued kernels
+  release->store(true, cuda::memory_order_release);
+
+  for (size_t i = 0; i < dev_count; ++i) {
+    CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
+    CUDA_CHECK(cudaStreamSynchronize(streams[i]));
+  }
+
+  result.runtime = 0;
+  for (size_t i = 0; i < dev_count; ++i) {
+    CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
+    result.runtime_vector[i] = timer[i].duration(iterations);
+    result.runtime += result.runtime_vector[i];
+  }
+  result.runtime /= static_cast<double>(dev_count);
+
+  CUDA_CHECK(cudaFreeHost(release));
+
+  for (size_t i = 0; i < dev_count; ++i) {
+    CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
+    CUDA_CHECK(cudaGraphExecDestroy(graphExecs[i]));
+    CUDA_CHECK(cudaGraphDestroy(graphs[i]));
+  }
+
+  for (size_t i = 0; i < dev_count; ++i) {
+    CUDA_CHECK(cudaSetDevice(options.device.device_id(dev_count - i - 1)));
+    timer.pop_back();
+  }
+
+  return Status::kSuccess;
+}
+
+/// Method to profile GPU execution time of a kernel launched in func
+Status OperationProfiler::profile_kernel_(
+  PerformanceResult &result,
+  Options const &options,
+  const std::function<Status(cudaStream_t, int)> &func,
+  cudaStream_t stream) {
+
+  GpuTimer timer;
+  // Optional sleep to limit power consumption and thermals
+  sleep(options.profiling.sleep_duration);
+
+  Status status = Status::kSuccess;
+
+  int iterations;
+  status = predict_iters(iterations, options, func, stream);
+  if (status != Status::kSuccess) {
+    return status;
+  }
+
+  for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
+    status = func(stream, iteration);
+    if (status != Status::kSuccess) {
+      return status;
+    }
+  }
+
+  timer.start(stream);
+
+  int iteration = 0;
+  for (; iteration < iterations; ++iteration) {
+    status = func(stream, iteration + options.profiling.warmup_iterations);
+
+    if (status != Status::kSuccess) {
+      result.status = status;
+      return status;
+    }
+  }
+
+  timer.stop_and_wait(stream);
+
+  result.runtime = timer.duration(iteration);
+  result.status  = status;
+
+  return status;
+}
+
 /// Method to profile a CUTLASS Operation
 Status OperationProfiler::profile_cutlass_(
   PerformanceResult &result,
@@ -665,70 +873,8 @@ Status OperationProfiler::profile_cutlass_(
   void *host_workspace,
   void *device_workspace) {
 
-  GpuTimer timer;
-
-  //
-  // Optional sleep to limit power consumption and thermals
-  //
-
-  sleep(options.profiling.sleep_duration);
-
-  //
-  // Warmup loop
-  //
-
-  Status status;
-
-  for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
-
-    status = operation->run(
-      arguments,
-      host_workspace,
-      device_workspace);
-
-    if (status != Status::kSuccess) {
-      return status;
-    }
-  }
-
-  //
-  // Initialize GPU timer
-  //
-
-  timer.start();
-
-  //
-  // Profiling loop
-  //
-
-  int Iterations = options.profiling.iterations;
-
-  int iteration = 0;
-  for (; iteration < Iterations; ++iteration) {
-
-    status = operation->run(
-      arguments,
-      host_workspace,
-      device_workspace);
-
-    if (status != Status::kSuccess) {
-      return status;
-    }
-  }
-
-  //
-  // Wait for completion
-  //
-
-  timer.stop_and_wait();
-
-  //
-  // Update performance result
-  //
-
-  result.runtime = timer.duration(iteration);
-
-  return status;
+  auto op = [=](cudaStream_t, int) { return operation->run(arguments, host_workspace, device_workspace); };
+  return profile_kernel_(result, options, op);
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////
@@ -748,9 +894,6 @@ void OperationProfiler::initialize_result_(
   set_argument(result, "cta_m", problem_space, operation_desc.tile_description.threadblock_shape.m());
   set_argument(result, "cta_n", problem_space, operation_desc.tile_description.threadblock_shape.n());
   set_argument(result, "cta_k", problem_space, operation_desc.tile_description.threadblock_shape.k());
-  set_argument(result, "cluster_m", problem_space, operation_desc.tile_description.cluster_shape.m());
-  set_argument(result, "cluster_n", problem_space, operation_desc.tile_description.cluster_shape.n());
-  set_argument(result, "cluster_k", problem_space, operation_desc.tile_description.cluster_shape.k());
   set_argument(result, "stages", problem_space, operation_desc.tile_description.threadblock_stages);
   set_argument(result, "warps_m", problem_space, operation_desc.tile_description.warp_count.m());
   set_argument(result, "warps_n", problem_space, operation_desc.tile_description.warp_count.n());

tools/profiler/src/options.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without
@@ -474,6 +474,8 @@ Options::Profiling::Profiling(cutlass::CommandLine const &cmdline) {
   cmdline.get_cmd_line_argument("profiling-iterations", iterations, 100);
   cmdline.get_cmd_line_argument("sleep-duration", sleep_duration, 50);
   cmdline.get_cmd_line_argument("profiling-enabled", enabled, true);
+  cmdline.get_cmd_line_argument("profiling-duration", duration, 10);
+  cmdline.get_cmd_line_argument("min-iterations", min_iterations, 10);
 
   if (cmdline.check_cmd_line_flag("providers")) {
 
@@ -504,7 +506,17 @@ void Options::Profiling::print_usage(std::ostream &out) const {
 
     << "  --profiling-iterations=<iterations>          "
     << "    Number of iterations to profile each kernel. If zero, kernels" << end_of_line
-    << "      are launched up to the profiling duration.\n\n"
+    << "      are launched up to the profiling duration. If non-zero, this overrides" << end_of_line
+    << "      --profiling-duration and --min-iterations.\n\n"
+
+    << "  --profiling-duration=<duration>             "
+    << "    Time to spend profiling each kernel (ms)." << end_of_line
+    << "    Overriden by `profiling-iterations` when `profiling-iterations` > 0." << end_of_line
+    << "    Note that `min-iterations` must also be satisfied.\n\n"
+
+    << "  --min-iterations=<iterations>             "
+    << "    Minimum number of iterations to spend profiling each kernel, even if" << end_of_line
+    << "    `profiling-duration` has been met.\n\n"
 
     << "  --warmup-iterations=<iterations>             "
     << "    Number of iterations to execute each kernel prior to profiling.\n\n"

tools/profiler/src/performance_report.cpp

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without

tools/profiler/src/performance_result.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without

tools/profiler/src/problem_space.cpp

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without
@@ -800,6 +800,44 @@ bool arg_as_int(
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
 
+/// Lexically casts an argument to an bool if it is defined. Returns true if not null.
+bool arg_as_bool(bool &bool_value, KernelArgument::Value const *value_ptr) {
+  if (value_ptr->not_null) {
+    if (value_ptr->argument->description->type == ArgumentTypeID::kInteger) {
+      int64_t value64;
+      arg_as_int(value64, value_ptr);
+      bool_value = static_cast<bool>(value64); 
+    }
+    else if (value_ptr->argument->description->type == ArgumentTypeID::kEnumerated) {
+      bool_value = library::from_string<bool>(
+        static_cast<EnumeratedTypeArgument::EnumeratedTypeValue const *>(value_ptr)->element);
+    }
+    else {
+      throw std::runtime_error(
+        "arg_as_bool() - illegal cast. Problem space argument must be integer or enumerated");
+    }
+
+    return true;
+  }
+
+  return false;
+}
+
+/// Lexically casts an argument to a bool
+bool arg_as_bool(
+  bool &bool_value,
+  char const *name,
+  ProblemSpace const &problem_space,
+  ProblemSpace::Problem const &problem) {
+
+  size_t idx = problem_space.argument_index(name);
+  KernelArgument::Value const *value_ptr = problem.at(idx).get();
+
+  return arg_as_bool(bool_value, value_ptr);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
 /// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
 bool arg_as_NumericTypeID(
   library::NumericTypeID &numeric_type, 

tools/profiler/src/rank_2k_operation_profiler.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without
@@ -297,6 +297,10 @@ void Rank2KOperationProfiler::RankKProblem::initialize_result(
   set_argument(result, "n", problem_space, n);
   set_argument(result, "k", problem_space, k);
 
+  set_argument(result, "cluster_m", problem_space, operation_desc.tile_description.cluster_shape.m());
+  set_argument(result, "cluster_n", problem_space, operation_desc.tile_description.cluster_shape.n());
+  set_argument(result, "cluster_k", problem_space, operation_desc.tile_description.cluster_shape.k());
+
   set_argument(result, "split_k_slices", problem_space, split_k_slices);
   set_argument(result, "batch_count", problem_space, batch_count);
 

tools/profiler/src/rank_k_operation_profiler.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without
@@ -286,6 +286,10 @@ void RankKOperationProfiler::RankKProblem::initialize_result(
   set_argument(result, "n", problem_space, n);
   set_argument(result, "k", problem_space, k);
 
+  set_argument(result, "cluster_m", problem_space, operation_desc.tile_description.cluster_shape.m());
+  set_argument(result, "cluster_n", problem_space, operation_desc.tile_description.cluster_shape.n());
+  set_argument(result, "cluster_k", problem_space, operation_desc.tile_description.cluster_shape.k());
+
   set_argument(result, "split_k_slices", problem_space, split_k_slices);
   set_argument(result, "batch_count", problem_space, batch_count);
 

tools/profiler/src/sparse_gemm_operation_profiler.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without
@@ -240,6 +240,10 @@ void SparseGemmOperationProfiler::SparseGemmProblem::initialize_result(
   set_argument(result, "n", problem_space, n);
   set_argument(result, "k", problem_space, k);
 
+  set_argument(result, "cluster_m", problem_space, operation_desc.tile_description.cluster_shape.m());
+  set_argument(result, "cluster_n", problem_space, operation_desc.tile_description.cluster_shape.n());
+  set_argument(result, "cluster_k", problem_space, operation_desc.tile_description.cluster_shape.k());
+
   set_argument(result, "split_k_slices", problem_space, split_k_slices);
   set_argument(result, "batch_count", problem_space, batch_count);
 

tools/profiler/src/symm_operation_profiler.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without
@@ -320,6 +320,10 @@ void SymmOperationProfiler::SymmProblem::initialize_result(
   set_argument(result, "m", problem_space, m);
   set_argument(result, "n", problem_space, n);
 
+  set_argument(result, "cluster_m", problem_space, operation_desc.tile_description.cluster_shape.m());
+  set_argument(result, "cluster_n", problem_space, operation_desc.tile_description.cluster_shape.n());
+  set_argument(result, "cluster_k", problem_space, operation_desc.tile_description.cluster_shape.k());
+
   set_argument(result, "split_k_slices", problem_space, split_k_slices);
   set_argument(result, "batch_count", problem_space, batch_count);
 

tools/profiler/src/trmm_operation_profiler.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Redistribution and use in source and binary forms, with or without
@@ -251,6 +251,10 @@ void TrmmOperationProfiler::TrmmProblem::initialize_result(
   set_argument(result, "m", problem_space, m);
   set_argument(result, "n", problem_space, n);
 
+  set_argument(result, "cluster_m", problem_space, operation_desc.tile_description.cluster_shape.m());
+  set_argument(result, "cluster_n", problem_space, operation_desc.tile_description.cluster_shape.n());
+  set_argument(result, "cluster_k", problem_space, operation_desc.tile_description.cluster_shape.k());
+
   set_argument(result, "split_k_slices", problem_space, split_k_slices);
   set_argument(result, "batch_count", problem_space, batch_count);