Merge remote-tracking branch 'origin/master' into small_alignment

include/cutlass/conv/conv2d_problem_size.h

@@ -47,6 +47,7 @@
 #include "cutlass/gemm/gemm.h"
 #include "cutlass/matrix_coord.h"
 #include "cutlass/conv/convolution.h"
+#include "cutlass/functional.h"
 
 namespace cutlass {
 namespace conv {
@@ -485,6 +486,27 @@ int strided_dgrad_tile_m_per_filter(
   return tile_m_per_filter;
 }
 
+// Computes starting Dx coord (h, w) for given starting filter postion
+CUTLASS_HOST_DEVICE
+void strided_dgrad_starting_coords(
+  Conv2dProblemSize const &problem_size,
+  FastDivmod const &stride_h_divmod, FastDivmod const &stride_w_divmod,
+  int r, int s,
+  int &start_h, int &start_w) {
+
+  // function locals for remainder by fast divmod
+  int pad_h_rem_, pad_w_rem_;
+
+  // start_h  = std::abs(problem_size.stride_h - ((problem_size.pad_h % problem_size.stride_h) - r)) % problem_size.stride_h;
+  stride_h_divmod.divmod(pad_h_rem_, problem_size.pad_h);
+  int r_ = std::abs(problem_size.stride_h - (pad_h_rem_ - r));
+  stride_h_divmod.divmod(start_h, r_);
+
+  //start_w  = std::abs(problem_size.stride_w - ((problem_size.pad_w % problem_size.stride_w) - s)) % problem_size.stride_w;
+  stride_w_divmod.divmod(pad_w_rem_, problem_size.pad_w);
+  int s_ = std::abs(problem_size.stride_w - (pad_w_rem_ - s));
+  stride_w_divmod.divmod(start_w, s_);
+}
 
 } // namespace conv
 } // namespace cutlass

include/cutlass/conv/device/implicit_gemm_convolution.h

@@ -217,14 +217,6 @@ public:
       if (result != cudaSuccess) {
         return Status::kErrorInternal;
       }
-
-      result = cudaFuncSetAttribute(
-          cutlass::Kernel<ImplicitGemmKernel>,
-          cudaFuncAttributePreferredSharedMemoryCarveout, 100);
-
-      if (result != cudaSuccess) {
-        return Status::kErrorInternal;
-      }
     }
     
     return Status::kSuccess;

include/cutlass/conv/kernel/implicit_gemm_convolution_strided_dgrad.h

@@ -199,7 +199,8 @@ struct ImplicitGemmConvolutionStridedDgrad {
   struct Params {
     ConvProblemSize problem_size;
     cutlass::gemm::GemmCoord grid_tiled_shape;
-    FastDivmod filter_s_divmod;
+    FastDivmod stride_h_divmod;
+    FastDivmod stride_w_divmod;
     int gemm_k_iterations;
     typename Mma::IteratorA::Params iterator_A;
     typename Mma::IteratorA::Element const *ptr_A;
@@ -227,7 +228,8 @@ struct ImplicitGemmConvolutionStridedDgrad {
       int *semaphore = nullptr
     ):
       problem_size(args.problem_size),
-      filter_s_divmod(args.problem_size.stride_w),
+      stride_h_divmod(args.problem_size.stride_h),
+      stride_w_divmod(args.problem_size.stride_w),
       iterator_A(Mma::IteratorA::getParams(args.problem_size, args.ref_A.layout())),
       ptr_A(args.ref_A.data()),
       iterator_B(args.problem_size, args.ref_B.layout()),
@@ -297,7 +299,7 @@ struct ImplicitGemmConvolutionStridedDgrad {
     // int start_s = filter_tile_m % (params.problem_size.stride_w);
 
     int start_r, start_s;
-    params.filter_s_divmod(start_r, start_s, filter_tile_m);
+    params.stride_w_divmod(start_r, start_s, filter_tile_m);
 
     typename Mma::FragmentC accumulators;
 
@@ -320,6 +322,7 @@ struct ImplicitGemmConvolutionStridedDgrad {
         params.problem_size,
         params.ptr_A,
         thread_idx,
+        params.stride_h_divmod, params.stride_w_divmod,
         start_r, start_s,
         MatrixCoord(
           threadblock_tile_idx.m() * Mma::Shape::kM,
@@ -386,6 +389,7 @@ struct ImplicitGemmConvolutionStridedDgrad {
       params.ptr_D,
       ConvOutputIteratorParameter::extent(params.problem_size),
       thread_idx,
+      params.stride_h_divmod, params.stride_w_divmod,
       start_r, start_s,
       threadblock_offset
     );
@@ -396,6 +400,7 @@ struct ImplicitGemmConvolutionStridedDgrad {
       params.ptr_C,
       ConvOutputIteratorParameter::extent(params.problem_size),
       thread_idx,
+      params.stride_h_divmod, params.stride_w_divmod,
       start_r, start_s,
       threadblock_offset
     );

include/cutlass/conv/threadblock/conv2d_dgrad_output_gradient_tile_access_iterator_analytic.h

@@ -132,7 +132,6 @@ private:
   int offset_p_[ThreadMap::Iterations::kStrided];
   int offset_q_[ThreadMap::Iterations::kStrided];
 
-
 public:
 
   CUTLASS_HOST_DEVICE
@@ -141,6 +140,7 @@ public:
     Conv2dProblemSize const &problem_size,
     Element const *ptr,
     int thread_idx,
+    FastDivmod const &stride_h_divmod, FastDivmod const &stride_w_divmod,
     int start_r, int start_s,
     MatrixCoord const &threadblock_offset = MatrixCoord()     // threadblock offset - units are whole CTA tiles
   ):
@@ -166,9 +166,12 @@ public:
     }
 
     // Starting h, w positions for filter position in gemm_k=0
-    int start_h = std::abs((problem_size_.pad_h - filter_r) % problem_size_.stride_h);
-    int start_w = std::abs((problem_size_.pad_w - filter_s) % problem_size_.stride_w);
-
+    int start_h, start_w;
+    strided_dgrad_starting_coords(
+      problem_size_, 
+      stride_h_divmod, stride_w_divmod, 
+      filter_r, filter_s, 
+      start_h, start_w);
 
     // Effective P and Q for filter position required for remapping NHW rows
     int P = (problem_size_.H - start_h + problem_size_.stride_h - 1) / problem_size_.stride_h;

include/cutlass/conv/threadblock/conv2d_tile_iterator.h

@@ -203,7 +203,27 @@ private:
 
 public:
 
-  /// Constructor
+  /// Constructor (output gradient (Dy) OperandA ctor)
+  CUTLASS_HOST_DEVICE
+  TileIteratorStridedDgrad(
+    Params const &params,
+    ConvProblemSize const &problem_size,
+    Element const *ptr,
+    int thread_idx,
+    FastDivmod const &stride_h_divmod, FastDivmod const &stride_w_divmod,
+    int start_r, int start_s,
+    MatrixCoord const &threadblock_offset = MatrixCoord()
+  ):
+    tile_access_iterator_(
+      params, 
+      problem_size, 
+      ptr, 
+      thread_idx, 
+      stride_h_divmod, stride_w_divmod, 
+      start_r, start_s, 
+      threadblock_offset) { }
+
+  /// Constructor (filter (w) OperandB ctor)
   CUTLASS_HOST_DEVICE
   TileIteratorStridedDgrad(
     Params const &params,
@@ -213,7 +233,12 @@ public:
     int start_r, int start_s,
     MatrixCoord const &threadblock_offset = MatrixCoord()
   ):
-    tile_access_iterator_(params, problem_size, ptr, thread_idx, start_r, start_s, threadblock_offset) { }
+    tile_access_iterator_(params, 
+      problem_size, 
+      ptr, 
+      thread_idx, 
+      start_r, start_s, 
+      threadblock_offset) { }
 
   CUTLASS_HOST_DEVICE
   static Params getParams(ConvProblemSize const &problem_size, Layout const &layout) {