3.6.0 update (#2005)

* 3.6.0 update

* doc and swap stuff

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>

tools/profiler/include/cutlass/profiler/conv2d_operation_profiler.h

@@ -430,7 +430,7 @@ public:
 protected:
   /// Method to profile an initialized CUTLASS operation
   virtual Status profile_cutlass_(
-    double &runtime,
+    PerformanceResult &result,
     Options const &options,
     library::Operation const *operation,
     void *arguments,

tools/profiler/include/cutlass/profiler/conv3d_operation_profiler.h

@@ -384,7 +384,7 @@ protected:
 
   /// Method to profile an initialized CUTLASS operation
   virtual Status profile_cutlass_(
-    double &runtime,
+    PerformanceResult &result,
     Options const &options,
     library::Operation const *operation,
     void *arguments,

tools/profiler/include/cutlass/profiler/cublas_helpers.h

@@ -304,7 +304,7 @@ struct cublasLtGemmExDispatcher {
     ); 
 
   /// Executes GEMM using these arguments
-  cublasStatus_t operator()(cublasLtHandle_t handle);
+  cublasStatus_t operator()(cublasLtHandle_t handle, cudaStream_t stream = nullptr);
 
   ~cublasLtGemmExDispatcher(){
 

tools/profiler/include/cutlass/profiler/enumerated_types.h

@@ -90,9 +90,9 @@ AlgorithmMode from_string<AlgorithmMode>(std::string const &str);
 /// Outcome of a performance test
 enum class Disposition {
   kPassed,
-  kFailed,
+  kFailed,    // kernel itself reported an error
   kNotRun,
-  kIncorrect,
+  kIncorrect, // kernel finished without a detected error, but result does not equal expected result
   kNotVerified,
   kInvalidProblem,
   kNotSupported,

tools/profiler/include/cutlass/profiler/gemm_operation_profiler.h

@@ -143,6 +143,8 @@ public:
 
     /// Buffer used for the cutlass reduction operations' host workspace
     std::vector<uint8_t> reduction_host_workspace;
+
+    cudaStream_t stream;
   };
 
 protected:
@@ -155,7 +157,7 @@ protected:
   GemmProblem problem_;
 
   /// Device memory allocations
-  GemmWorkspace gemm_workspace_;
+  std::vector<GemmWorkspace> gemm_workspace_;
 
   /// CUTLASS parallel reduction operation to follow this* gemm operation
   library::Operation const *reduction_op_;
@@ -231,7 +233,8 @@ protected:
     DeviceContext &device_context,
     library::Operation const *operation,
     ProblemSpace const &problem_space,
-    ProblemSpace::Problem const &problem);
+    ProblemSpace::Problem const &problem,
+    GemmWorkspace &gemm_workspace);
 
   /// Verifies CUTLASS against host and device references
   bool verify_with_reference_(
@@ -246,7 +249,7 @@ protected:
 
   /// Method to profile a CUTLASS Operation
   Status profile_cutlass_(
-    double &runtime,
+    PerformanceResult &result,
     Options const &options,
     library::Operation const *operation,
     void *arguments,

tools/profiler/include/cutlass/profiler/gpu_timer.h

@@ -51,16 +51,21 @@ struct GpuTimer {
   //
   
   GpuTimer();
+
+  GpuTimer(GpuTimer const&) = delete;
+
+  GpuTimer(GpuTimer &&gpu_timer) noexcept;
+
   ~GpuTimer();
 
-  /// Records a start event in the stream
-  void start(cudaStream_t stream = nullptr);
+  /// Records a start event in the stream, the flag is for cudaEventRecordWithFlags
+  void start(cudaStream_t stream = nullptr, unsigned int flag = cudaEventRecordDefault);
 
-  /// Records a stop event in the stream
-  void stop(cudaStream_t stream = nullptr);
+  /// Records a stop event in the stream, the flag is for cudaEventRecordWithFlags
+  void stop(cudaStream_t stream = nullptr, unsigned int flag = cudaEventRecordDefault);
 
-  /// Records a stop event in the stream and synchronizes on the stream
-  void stop_and_wait(cudaStream_t stream = nullptr);
+  /// Records a stop event in the stream and synchronizes on the stream, the flag is for cudaEventRecordWithFlags
+  void stop_and_wait(cudaStream_t stream = nullptr, unsigned int flag = cudaEventRecordDefault);
 
   /// Returns the duration in milliseconds
   double duration(int iterations = 1) const;

tools/profiler/include/cutlass/profiler/operation_profiler.h

@@ -232,7 +232,7 @@ protected:
 
   /// Method to profile an initialized CUTLASS operation
   virtual Status profile_cutlass_(
-    double &runtime,
+    PerformanceResult &result,
     Options const &options,
     library::Operation const *operation,
     void *arguments,

tools/profiler/include/cutlass/profiler/performance_result.h

@@ -86,6 +86,9 @@ struct PerformanceResult {
   /// Average runtime in ms
   double runtime;
 
+  /// Average runtime in ms per device
+  std::vector<double> runtime_vector;
+
   //
   // Members
   //