CUTLASS 2.6.1 - functional and performance enhancements to strided DGRAD, fixes, and tuning

* cutlass 2.6 update

* remove debug prints

* cutlass 2.6.1 (minor update)

* Updated CHANGELOG.

* Minor edit to readme to indicate patch version.

* Minor edit to readme.

Co-authored-by:  Haicheng Wu <haichengw@nvidia.com>, Andrew Kerr <akerr@nvidia.com>

include/cutlass/epilogue/threadblock/default_epilogue_tensor_op.h

@@ -309,9 +309,12 @@ struct DefaultEpilogueTensorOp {
     kElementsPerAccess
   >::Type;
 
+  static bool const UseCUDAStore = platform::is_same<ElementOutput, double>::value;
+
   using OutputTileIterator = cutlass::epilogue::threadblock::PredicatedTileIterator<
     OutputTileThreadMap,
-    ElementOutput
+    ElementOutput,
+    UseCUDAStore
   >;
 
   using AccumulatorFragmentIterator = typename std::conditional<is_complex<ElementOutput>::value,

include/cutlass/epilogue/threadblock/predicated_tile_iterator.h

@@ -62,7 +62,8 @@ namespace threadblock {
 ///
 template <
   typename ThreadMap_,       ///< Thread map (conept: OutputTileThreadMap)
-  typename Element_          ///< Element data type
+  typename Element_,         ///< Element data type
+  bool UseCUDAStore = false
 >
 class PredicatedTileIterator {
 public:
@@ -341,10 +342,17 @@ public:
 
             bool guard = row_guard && mask_.predicates[column];
 
-            cutlass::arch::global_store<AccessType, sizeof(AccessType)>(
-                frag_ptr[frag_row_idx * ThreadMap::Iterations::kColumn + column],
-                (void *)&memory_pointer[column * ThreadMap::Delta::kColumn / kElementsPerAccess],
-                guard);
+            if (UseCUDAStore) {
+              if (guard) {
+                memory_pointer[column * ThreadMap::Delta::kColumn / kElementsPerAccess] =
+                    frag_ptr[frag_row_idx * ThreadMap::Iterations::kColumn + column];
+              }
+            } else {
+              cutlass::arch::global_store<AccessType, sizeof(AccessType)>(
+                  frag_ptr[frag_row_idx * ThreadMap::Iterations::kColumn + column],
+                  (void *)&memory_pointer[column * ThreadMap::Delta::kColumn / kElementsPerAccess],
+                  guard);
+            }
           }
 
           if (row + 1 < ThreadMap::Iterations::kRow) {

include/cutlass/epilogue/threadblock/predicated_tile_iterator_strided_dgrad.h

@@ -222,6 +222,7 @@ public:
     Element *pointer,
     TensorCoord extent,
     int thread_idx,
+    FastDivmod const &stride_h_divmod, FastDivmod const &stride_w_divmod,
     int start_r, int start_s,
     TensorCoord threadblock_offset = TensorCoord()
   ): 
@@ -238,9 +239,12 @@ public:
       s = (params_.problem_size.S - 1 - s);
     }
 
-    // check if start_h_ and start_w_ are always positive
-    start_h_ = std::abs((params_.problem_size.pad_h - r) % params_.problem_size.stride_h);
-    start_w_ = std::abs((params_.problem_size.pad_w - s) % params_.problem_size.stride_w);
+    // compute starting coordinates in Dx start_h_ and start_w_
+    strided_dgrad_starting_coords(
+      params_.problem_size, 
+      stride_h_divmod, stride_w_divmod, 
+      r, s, 
+      start_h_, start_w_);
 
     p_ = (params_.problem_size.H - start_h_ + params_.problem_size.stride_h - 1) / params_.problem_size.stride_h;
     q_ = (params_.problem_size.W - start_w_ + params_.problem_size.stride_w - 1) / params_.problem_size.stride_w;