CUTLASS 3.7 (#2045)

* CUTLASS 3.7 * clean up changelog --------- Co-authored-by: yuzhai <yuzhai@nvidia.com> Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
2025-01-18 09:53:07 -05:00
parent 902dff3663
commit b78588d163
2030 changed files with 8947 additions and 3475 deletions
@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: BSD-3-Clause
 *
 * Redistribution and use in source and binary forms, with or without
@@ -47,6 +47,8 @@
 // sleep not supported
 #endif

+#include <cuda/atomic>
+
 #include "cutlass/profiler/options.h"
 #include "cutlass/profiler/operation_profiler.h"
 #include "cutlass/profiler/gpu_timer.h"
@@ -55,9 +57,18 @@

 ///////////////////////////////////////////////////////////////////////////////////////////////////

+#define CUDA_CHECK(call)                                                                                               \
+  do {                                                                                                                 \
+    cudaError_t err = call;                                                                                            \
+    if (err != cudaSuccess) {                                                                                          \
+      std::cerr << "CUDA error at " << __FILE__ << ":" << __LINE__ << " code=" << err << " \""                         \
+                << cudaGetErrorString(err) << "\"\n";                                                                  \
+      return Status::kErrorInternal;                                                                                   \
+    }                                                                                                                  \
+  } while (0)
+
 namespace cutlass {
 namespace profiler {
-
 ///////////////////////////////////////////////////////////////////////////////////////////////////

 OperationProfiler::OperationProfiler(): kind_(library::OperationKind::kInvalid) { }
@@ -656,6 +667,203 @@ void OperationProfiler::save_workspace(

 ///////////////////////////////////////////////////////////////////////////////////////////////////

+namespace {
+extern "C" {
+__global__ void delay(cuda::atomic<bool> const *release) {
+  while (release->load(cuda::memory_order_acquire) != true) {
+#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 700)
+    __nanosleep(100);
+#endif
+  }
+}
+}
+
+Status predict_iters(
+  int &iterations,
+  Options const &options,
+  const std::function<Status(cudaStream_t, int)> &func,
+  cudaStream_t stream) {
+  // always use profiling-iterations if requested
+  if (options.profiling.iterations != 0) {
+    iterations = options.profiling.iterations;
+    return Status::kSuccess;
+  }
+
+  // otherwise run for as many iterations as necessary to
+  // meet profiling-duration
+  constexpr int CALIBRATION_ITERS = 5;
+  GpuTimer timer;
+  timer.start(stream);
+  for (int i = 0; i < CALIBRATION_ITERS; i++) {
+    Status status = func(stream, i);
+    if (status != Status::kSuccess) {
+      return status;
+    }
+  }
+  timer.stop_and_wait(stream);
+
+  double est_iters             = options.profiling.duration / std::max(timer.duration(CALIBRATION_ITERS), 1e-6);
+  constexpr uint64_t MAX_ITERS = 1'000'000;
+  iterations = std::min(static_cast<uint64_t>(std::ceil(est_iters)), static_cast<uint64_t>(MAX_ITERS));
+  iterations = std::max(options.profiling.min_iterations, iterations);
+  return Status::kSuccess;
+};
+
+} // namespace
+
+/// This profiling method is designed to run a kernel on several GPUs to
+/// measure interference (e.g. due to power throttling).
+/// To encourage the kernels to start at the same time and minimize jitter,
+/// a spinloop kernel blocks each stream while work is being enqueued, which is
+/// later triggered from the host.
+/// CUDA graphs allows you to record the launch of large numbers of kernels without
+/// blocking and therefore avoids a deadlock which happens if you try to enqueue too
+/// many kernels behind the spinloop kernel.
+Status OperationProfiler::profile_kernel_(
+  PerformanceResult &result,
+  Options const &options,
+  const std::function<Status(int, cudaStream_t, int)> &func,
+  const std::vector<cudaStream_t> &streams) {
+  auto dev_count = streams.size();
+  cuda::atomic<bool> *release;
+  CUDA_CHECK(cudaHostAlloc(&release, sizeof(*release), cudaHostAllocPortable));
+  release->store(false, cuda::memory_order_release);
+
+  std::vector<GpuTimer> timer;
+  for (size_t i = 0; i < dev_count; ++i) {
+    CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
+    timer.emplace_back();
+  }
+
+  std::vector<cudaGraph_t> graphs;
+  graphs.resize(dev_count);
+  std::vector<cudaGraphExec_t> graphExecs;
+  graphExecs.resize(dev_count);
+
+  sleep(options.profiling.sleep_duration);
+
+  // predict time by running on device 0
+  int iterations;
+  CUDA_CHECK(cudaSetDevice(0));
+  Status status = predict_iters(
+    iterations,
+    options,
+    [&](cudaStream_t stream, int iter) { return func(0, stream, iter); },
+    streams[0]);
+  if (status != Status::kSuccess) {
+    return status;
+  }
+
+  for (size_t i = 0; i < dev_count; ++i) {
+    CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
+    CUDA_CHECK(cudaStreamBeginCapture(streams[i], cudaStreamCaptureModeGlobal));
+    // Halt execution until all GPUs are ready to precede.
+    // It allows the CPU to trigger the GPUs all start at the same time.
+    delay<<<1, 1, 0, streams[i]>>>(release);
+    for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
+      Status status = func(i, streams[i], iteration);
+      if (status != Status::kSuccess) {
+        return status;
+      }
+    }
+
+    timer[i].start(streams[i], cudaEventRecordExternal);
+
+    int iteration = 0;
+    for (; iteration < iterations; ++iteration) {
+      Status status = func(i, streams[i], iteration + options.profiling.warmup_iterations);
+      if (status != Status::kSuccess) {
+        return status;
+      }
+    }
+    timer[i].stop(streams[i], cudaEventRecordExternal);
+    CUDA_CHECK(cudaStreamEndCapture(streams[i], &graphs[i]));
+    CUDA_CHECK(cudaGraphInstantiate(&graphExecs[i], graphs[i], nullptr, nullptr, 0));
+  }
+
+  for (size_t i = 0; i < dev_count; ++i) {
+    CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
+    CUDA_CHECK(cudaGraphLaunch(graphExecs[i], streams[i]));
+  }
+
+  // release the enqueued kernels
+  release->store(true, cuda::memory_order_release);
+
+  for (size_t i = 0; i < dev_count; ++i) {
+    CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
+    CUDA_CHECK(cudaStreamSynchronize(streams[i]));
+  }
+
+  result.runtime = 0;
+  for (size_t i = 0; i < dev_count; ++i) {
+    CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
+    result.runtime_vector[i] = timer[i].duration(iterations);
+    result.runtime += result.runtime_vector[i];
+  }
+  result.runtime /= static_cast<double>(dev_count);
+
+  CUDA_CHECK(cudaFreeHost(release));
+
+  for (size_t i = 0; i < dev_count; ++i) {
+    CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
+    CUDA_CHECK(cudaGraphExecDestroy(graphExecs[i]));
+    CUDA_CHECK(cudaGraphDestroy(graphs[i]));
+  }
+
+  for (size_t i = 0; i < dev_count; ++i) {
+    CUDA_CHECK(cudaSetDevice(options.device.device_id(dev_count - i - 1)));
+    timer.pop_back();
+  }
+
+  return Status::kSuccess;
+}
+
+/// Method to profile GPU execution time of a kernel launched in func
+Status OperationProfiler::profile_kernel_(
+  PerformanceResult &result,
+  Options const &options,
+  const std::function<Status(cudaStream_t, int)> &func,
+  cudaStream_t stream) {
+
+  GpuTimer timer;
+  // Optional sleep to limit power consumption and thermals
+  sleep(options.profiling.sleep_duration);
+
+  Status status = Status::kSuccess;
+
+  int iterations;
+  status = predict_iters(iterations, options, func, stream);
+  if (status != Status::kSuccess) {
+    return status;
+  }
+
+  for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
+    status = func(stream, iteration);
+    if (status != Status::kSuccess) {
+      return status;
+    }
+  }
+
+  timer.start(stream);
+
+  int iteration = 0;
+  for (; iteration < iterations; ++iteration) {
+    status = func(stream, iteration + options.profiling.warmup_iterations);
+
+    if (status != Status::kSuccess) {
+      result.status = status;
+      return status;
+    }
+  }
+
+  timer.stop_and_wait(stream);
+
+  result.runtime = timer.duration(iteration);
+  result.status  = status;
+
+  return status;
+}
+
 /// Method to profile a CUTLASS Operation
 Status OperationProfiler::profile_cutlass_(
  PerformanceResult &result,
@@ -665,70 +873,8 @@ Status OperationProfiler::profile_cutlass_(
  void *host_workspace,
  void *device_workspace) {

-  GpuTimer timer;
-
-  //
-  // Optional sleep to limit power consumption and thermals
-  //
-
-  sleep(options.profiling.sleep_duration);
-
-  //
-  // Warmup loop
-  //
-
-  Status status;
-
-  for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
-
-    status = operation->run(
-      arguments,
-      host_workspace,
-      device_workspace);
-
-    if (status != Status::kSuccess) {
-      return status;
-    }
-  }
-
-  //
-  // Initialize GPU timer
-  //
-
-  timer.start();
-
-  //
-  // Profiling loop
-  //
-
-  int Iterations = options.profiling.iterations;
-
-  int iteration = 0;
-  for (; iteration < Iterations; ++iteration) {
-
-    status = operation->run(
-      arguments,
-      host_workspace,
-      device_workspace);
-
-    if (status != Status::kSuccess) {
-      return status;
-    }
-  }
-
-  //
-  // Wait for completion
-  //
-
-  timer.stop_and_wait();
-
-  //
-  // Update performance result
-  //
-
-  result.runtime = timer.duration(iteration);
-
-  return status;
+  auto op = [=](cudaStream_t, int) { return operation->run(arguments, host_workspace, device_workspace); };
+  return profile_kernel_(result, options, op);
 }

 ///////////////////////////////////////////////////////////////////////////////////////////////////
@@ -748,9 +894,6 @@ void OperationProfiler::initialize_result_(
  set_argument(result, "cta_m", problem_space, operation_desc.tile_description.threadblock_shape.m());
  set_argument(result, "cta_n", problem_space, operation_desc.tile_description.threadblock_shape.n());
  set_argument(result, "cta_k", problem_space, operation_desc.tile_description.threadblock_shape.k());
-  set_argument(result, "cluster_m", problem_space, operation_desc.tile_description.cluster_shape.m());
-  set_argument(result, "cluster_n", problem_space, operation_desc.tile_description.cluster_shape.n());
-  set_argument(result, "cluster_k", problem_space, operation_desc.tile_description.cluster_shape.k());
  set_argument(result, "stages", problem_space, operation_desc.tile_description.threadblock_stages);
  set_argument(result, "warps_m", problem_space, operation_desc.tile_description.warp_count.m());
  set_argument(result, "warps_n", problem_space, operation_desc.tile_description.warp_count.n());