CUTLASS 2.6 (#298)

CUTLASS 2.6

test/unit/conv/device/CMakeLists.txt

@@ -141,6 +141,9 @@ cutlass_test_unit_add_executable(
   conv2d_fprop_implicit_gemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32_sm75.cu
   conv2d_dgrad_implicit_gemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32_sm75.cu
   conv2d_wgrad_implicit_gemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32_sm75.cu
+
+  conv2d_fprop_with_broadcast_sm75.cu
+  conv2d_fprop_with_reduction_sm75.cu
 )
 
 if (CUTLASS_NVCC_MAX_ARCH GREATER_EQUAL 80)
@@ -158,15 +161,18 @@ if (CUTLASS_NVCC_MAX_ARCH GREATER_EQUAL 80)
   
   cutlass_test_unit_add_executable(
     cutlass_test_unit_conv_device_tensorop_f32_sm80
-  
+
     conv2d_fprop_implicit_gemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32_sm80.cu
     conv2d_dgrad_implicit_gemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32_sm80.cu
     conv2d_wgrad_implicit_gemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32_sm80.cu
   
     conv3d_wgrad_implicit_gemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32_sm75.cu
     conv3d_wgrad_implicit_gemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32_sm80.cu
+
+    # Strided Dgrad 
+    conv2d_strided_dgrad_implicit_gemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32_sm80.cu
   )
-  
+
   # Conv2d - TF32 input, F32 output, F32 accumulation
   
   cutlass_test_unit_add_executable(

test/unit/conv/device/conv2d_dgrad_implicit_gemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32_sm50.cu

@@ -71,7 +71,8 @@ TEST(SM50_Device_Conv2d_Dgrad_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_s
     cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
     2,
     cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
+    cutlass::conv::IteratorAlgorithm::kAnalytic,
+    cutlass::conv::StrideSupport::kUnity
   >::Kernel;
 
   using Conv2dDgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dDgradKernel>;

test/unit/conv/device/conv2d_dgrad_implicit_gemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32_sm80.cu

@@ -38,140 +38,6 @@
 
 #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
 
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Dgrad_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
-  32x64_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::complex<float>;
-  using ElementB           = cutlass::complex<float>;
-  using ElementC           = cutlass::complex<float>;
-  using ElementAccumulator = cutlass::complex<float>;
-  using ElementCompute     = cutlass::complex<float>;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dDgradKernel = typename cutlass::conv::kernel::DefaultConv2dDgrad<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<32, 64, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dDgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dDgradKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dDgrad>());
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Dgrad_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
-  64x64_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::complex<float>;
-  using ElementB           = cutlass::complex<float>;
-  using ElementC           = cutlass::complex<float>;
-  using ElementAccumulator = cutlass::complex<float>;
-  using ElementCompute     = cutlass::complex<float>;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dDgradKernel = typename cutlass::conv::kernel::DefaultConv2dDgrad<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<64, 64, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dDgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dDgradKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dDgrad>());
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Dgrad_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
-  128x128_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::complex<float>;
-  using ElementB           = cutlass::complex<float>;
-  using ElementC           = cutlass::complex<float>;
-  using ElementAccumulator = cutlass::complex<float>;
-  using ElementCompute     = cutlass::complex<float>;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dDgradKernel = typename cutlass::conv::kernel::DefaultConv2dDgrad<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<128, 128, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dDgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dDgradKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dDgrad>());
-
-}
 
 ////////////////////////////////////////////////////////////////////////////////
 TEST(SM80_Device_Conv2d_Dgrad_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
@@ -208,52 +74,7 @@ TEST(SM80_Device_Conv2d_Dgrad_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_s
     cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
     4,
     cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dDgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dDgradKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dDgrad>());
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Dgrad_Optimized_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
-  32x64_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::complex<float>;
-  using ElementB           = cutlass::complex<float>;
-  using ElementC           = cutlass::complex<float>;
-  using ElementAccumulator = cutlass::complex<float>;
-  using ElementCompute     = cutlass::complex<float>;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dDgradKernel = typename cutlass::conv::kernel::DefaultConv2dDgrad<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<32, 64, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kOptimized,
+    cutlass::conv::IteratorAlgorithm::kAnalytic,
     cutlass::conv::StrideSupport::kUnity
   >::Kernel;
 

test/unit/conv/device/conv2d_dgrad_implicit_gemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f16_sm80.cu

@@ -69,7 +69,8 @@ TEST(SM80_Device_Conv2d_Dgrad_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f16nhwc_tens
     cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
     3,
     cutlass::arch::OpMultiplyAdd,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
+    cutlass::conv::IteratorAlgorithm::kAnalytic,
+    cutlass::conv::StrideSupport::kUnity
   >::Kernel;
 
   using Conv2dDgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dDgradKernel>;

test/unit/conv/device/conv2d_dgrad_implicit_gemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32_sm70.cu

@@ -66,7 +66,9 @@ TEST(SM70_Device_Conv2d_Dgrad_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tens
     >,
     cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
     2,
-    cutlass::arch::OpMultiplyAdd
+    cutlass::arch::OpMultiplyAdd,
+    cutlass::conv::IteratorAlgorithm::kAnalytic,
+    cutlass::conv::StrideSupport::kUnity
   >::Kernel;
 
   using Conv2dDgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dDgradKernel>;

test/unit/conv/device/conv2d_dgrad_implicit_gemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32_sm75.cu

@@ -67,7 +67,9 @@ TEST(SM75_Device_Conv2d_Dgrad_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tens
     >,
     cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
     2,
-    cutlass::arch::OpMultiplyAdd
+    cutlass::arch::OpMultiplyAdd,
+    cutlass::conv::IteratorAlgorithm::kAnalytic,
+    cutlass::conv::StrideSupport::kUnity
   >::Kernel;
 
   using Conv2dDgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dDgradKernel>;

test/unit/conv/device/conv2d_dgrad_implicit_gemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32_sm80.cu

@@ -36,88 +36,6 @@
 
 #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
 
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Dgrad_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32,
-  128x128_32x3_64x64x32) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::half_t;
-  using ElementB           = cutlass::half_t;
-  using ElementC           = float;
-  using ElementAccumulator = float;
-  using ElementCompute     = float;
-
-  /// Device-level Conv2d instance
-  using Conv2dDgradKernel = typename cutlass::conv::kernel::DefaultConv2dDgrad<
-    ElementA, cutlass::layout::TensorNHWC,
-    ElementB, cutlass::layout::TensorNHWC,
-    ElementC, cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassTensorOp,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<128, 128, 32>,
-    cutlass::gemm::GemmShape<64, 64, 32>,
-    cutlass::gemm::GemmShape<16, 8, 16>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      128 / cutlass::sizeof_bits<ElementC>::value,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    3,
-    cutlass::arch::OpMultiplyAdd,
-    cutlass::conv::IteratorAlgorithm::kAnalytic,
-    cutlass::conv::StrideSupport::kStrided
-  >::Kernel;
-
-  using Conv2dDgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dDgradKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dDgrad>());
-}
-
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Dgrad_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32_unity_stride,
-  128x128_32x3_64x64x32) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::half_t;
-  using ElementB           = cutlass::half_t;
-  using ElementC           = float;
-  using ElementAccumulator = float;
-  using ElementCompute     = float;
-
-  /// Device-level Conv2d instance
-  using Conv2dDgradKernel = typename cutlass::conv::kernel::DefaultConv2dDgrad<
-    ElementA, cutlass::layout::TensorNHWC,
-    ElementB, cutlass::layout::TensorNHWC,
-    ElementC, cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassTensorOp,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<128, 128, 32>,
-    cutlass::gemm::GemmShape<64, 64, 32>,
-    cutlass::gemm::GemmShape<16, 8, 16>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      128 / cutlass::sizeof_bits<ElementC>::value,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    3,
-    cutlass::arch::OpMultiplyAdd,
-    cutlass::conv::IteratorAlgorithm::kAnalytic,
-    cutlass::conv::StrideSupport::kUnity
-  >::Kernel;
-
-  using Conv2dDgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dDgradKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dDgrad>());
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 TEST(SM80_Device_Conv2d_Dgrad_Optimized_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32_unity_stride,
   128x128_32x3_64x64x32) {
@@ -281,6 +199,5 @@ TEST(SM80_Device_Conv2d_Dgrad_Optimized_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_ten
   /// Run all unit test sizes with device-level Conv2d instance
   EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dDgrad>());
 }
-
 ////////////////////////////////////////////////////////////////////////////////
 #endif  // CUTLASS_ARCH_MMA_SM80_SUPPORTED

test/unit/conv/device/conv2d_dgrad_implicit_gemm_f32nhwc_f32nhwc_f32nhwc_simt_f32_sm80.cu

@@ -37,95 +37,6 @@
 
 #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
 
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Dgrad_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
-  32x64_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = float;
-  using ElementB           = float;
-  using ElementC           = float;
-  using ElementAccumulator = float;
-  using ElementCompute     = float;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dDgradKernel = typename cutlass::conv::kernel::DefaultConv2dDgrad<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<32, 64, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAdd,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dDgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dDgradKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dDgrad>());
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Dgrad_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
-  64x64_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = float;
-  using ElementB           = float;
-  using ElementC           = float;
-  using ElementAccumulator = float;
-  using ElementCompute     = float;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dDgradKernel = typename cutlass::conv::kernel::DefaultConv2dDgrad<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<64, 64, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAdd,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dDgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dDgradKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dDgrad>());
-
-}
 
 ////////////////////////////////////////////////////////////////////////////////
 TEST(SM80_Device_Conv2d_Dgrad_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
@@ -162,107 +73,7 @@ TEST(SM80_Device_Conv2d_Dgrad_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt
     cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
     4,
     cutlass::arch::OpMultiplyAdd,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dDgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dDgradKernel>;
-
-  test::conv::device::Conv2dProblemVector user_size;
-
-  user_size.push_back(cutlass::conv::Conv2dProblemSize(
-      {1, 8, 8, 4},   // input size  (NHWC)
-      {8, 1, 1, 4},   // filter size (KRSC)
-      {0, 0, 0, 0},                      // padding (pad_h, _, pad_w, _)
-      {1, 1},                            // stride (stride_h, stride_w)
-      {1, 1}                             // dilation (dilation_h, dilation_w) 
-    ));
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dDgrad>(user_size));
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Dgrad_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
-  128x128_8x4_64x32x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = float;
-  using ElementB           = float;
-  using ElementC           = float;
-  using ElementAccumulator = float;
-  using ElementCompute     = float;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dDgradKernel = typename cutlass::conv::kernel::DefaultConv2dDgrad<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<128, 128, 8>,
-    cutlass::gemm::GemmShape<64, 32, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAdd,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dDgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dDgradKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dDgrad>());
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Dgrad_Optimized_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
-  32x64_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = float;
-  using ElementB           = float;
-  using ElementC           = float;
-  using ElementAccumulator = float;
-  using ElementCompute     = float;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dDgradKernel = typename cutlass::conv::kernel::DefaultConv2dDgrad<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<32, 64, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAdd,
-    cutlass::conv::IteratorAlgorithm::kOptimized,
+    cutlass::conv::IteratorAlgorithm::kAnalytic,
     cutlass::conv::StrideSupport::kUnity
   >::Kernel;
 
@@ -273,6 +84,7 @@ TEST(SM80_Device_Conv2d_Dgrad_Optimized_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_sim
 
 }
 
+
 ////////////////////////////////////////////////////////////////////////////////
 TEST(SM80_Device_Conv2d_Dgrad_Optimized_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
   128x128_8x4_64x32x8) {

test/unit/conv/device/conv2d_fprop_implicit_gemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32_sm50.cu

@@ -130,93 +130,3 @@ TEST(SM50_Device_Conv2d_Fprop_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_s
 }
 
 ////////////////////////////////////////////////////////////////////////////////
-TEST(SM50_Device_Conv2d_Fprop_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
-  64x64_8x2_32x32x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::complex<float>;
-  using ElementB           = cutlass::complex<float>;
-  using ElementC           = cutlass::complex<float>;
-  using ElementAccumulator = cutlass::complex<float>;
-  using ElementCompute     = cutlass::complex<float>;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFprop<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm50,
-    cutlass::gemm::GemmShape<64, 64, 8>,
-    cutlass::gemm::GemmShape<32, 32, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<4>,
-    2,
-    cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM50_Device_Conv2d_Fprop_Optimized_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
-  32x64_8x2_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::complex<float>;
-  using ElementB           = cutlass::complex<float>;
-  using ElementC           = cutlass::complex<float>;
-  using ElementAccumulator = cutlass::complex<float>;
-  using ElementCompute     = cutlass::complex<float>;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFprop<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm50,
-    cutlass::gemm::GemmShape<32, 64, 8>,
-    cutlass::gemm::GemmShape<32, 32, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<4>,
-    2,
-    cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kOptimized
-  >::Kernel;
-
-  using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
-
-}
-
-

test/unit/conv/device/conv2d_fprop_implicit_gemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32_sm80.cu

@@ -37,184 +37,6 @@
 
 #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
 
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Fprop_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
-  32x64_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::complex<float>;
-  using ElementB           = cutlass::complex<float>;
-  using ElementC           = cutlass::complex<float>;
-  using ElementAccumulator = cutlass::complex<float>;
-  using ElementCompute     = cutlass::complex<float>;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFprop<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<32, 64, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Fprop_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
-  64x64_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::complex<float>;
-  using ElementB           = cutlass::complex<float>;
-  using ElementC           = cutlass::complex<float>;
-  using ElementAccumulator = cutlass::complex<float>;
-  using ElementCompute     = cutlass::complex<float>;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFprop<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<64, 64, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Fprop_Optimized_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
-  128x128_8x4_64x32x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::complex<float>;
-  using ElementB           = cutlass::complex<float>;
-  using ElementC           = cutlass::complex<float>;
-  using ElementAccumulator = cutlass::complex<float>;
-  using ElementCompute     = cutlass::complex<float>;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFprop<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<128, 128, 8>,
-    cutlass::gemm::GemmShape<64, 32, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kOptimized
-  >::Kernel;
-
-  using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
-}
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Fprop_Optimized_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
-  128x128_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::complex<float>;
-  using ElementB           = cutlass::complex<float>;
-  using ElementC           = cutlass::complex<float>;
-  using ElementAccumulator = cutlass::complex<float>;
-  using ElementCompute     = cutlass::complex<float>;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFprop<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<128, 128, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
-
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 TEST(SM80_Device_Conv2d_Fprop_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
   128x128_8x4_64x32x8) {
@@ -260,50 +82,6 @@ TEST(SM80_Device_Conv2d_Fprop_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_s
 
 }
 
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Fprop_Optimized_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
-  32x64_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::complex<float>;
-  using ElementB           = cutlass::complex<float>;
-  using ElementC           = cutlass::complex<float>;
-  using ElementAccumulator = cutlass::complex<float>;
-  using ElementCompute     = cutlass::complex<float>;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFprop<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<32, 64, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kOptimized
-  >::Kernel;
-
-  using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
-
-}
 
 ////////////////////////////////////////////////////////////////////////////////
 TEST(SM80_Device_Conv2d_Fprop_Optimized_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
@@ -348,50 +126,6 @@ TEST(SM80_Device_Conv2d_Fprop_Optimized_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_
   /// Run all unit test sizes with device-level Conv2d instance
   EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
 }
-////////////////////////////////////////////////////////////////////////////////
-
-TEST(SM80_Device_Conv2d_Fprop_Optimized_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
-  64x64_8x3_64x32x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::complex<float>;
-  using ElementB           = cutlass::complex<float>;
-  using ElementC           = cutlass::complex<float>;
-  using ElementAccumulator = cutlass::complex<float>;
-  using ElementCompute     = cutlass::complex<float>;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFprop<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<64, 64, 8>,
-    cutlass::gemm::GemmShape<64, 32, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    3,
-    cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kOptimized
-  >::Kernel;
-
-  using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
-}
 
 ////////////////////////////////////////////////////////////////////////////////
 #endif  // CUTLASS_ARCH_MMA_SM80_SUPPORTED

test/unit/conv/device/conv2d_fprop_implicit_gemm_f32nhwc_f32nhwc_f32nhwc_simt_f32_sm80.cu

@@ -37,96 +37,6 @@
 
 #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
 
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Fprop_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
-  32x64_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = float;
-  using ElementB           = float;
-  using ElementC           = float;
-  using ElementAccumulator = float;
-  using ElementCompute     = float;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFprop<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<32, 64, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAdd,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Fprop_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
-  64x64_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = float;
-  using ElementB           = float;
-  using ElementC           = float;
-  using ElementAccumulator = float;
-  using ElementCompute     = float;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFprop<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<64, 64, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAdd,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
-
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 TEST(SM80_Device_Conv2d_Fprop_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
   128x128_8x4_32x64x8) {
@@ -167,106 +77,6 @@ TEST(SM80_Device_Conv2d_Fprop_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt
 
   using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
 
-  test::conv::device::Conv2dProblemVector user_size;
-
-  user_size.push_back(cutlass::conv::Conv2dProblemSize(
-      {1, 8, 8, 4},   // input size  (NHWC)
-      {8, 1, 1, 4},   // filter size (KRSC)
-      {0, 0, 0, 0},                      // padding (pad_h, _, pad_w, _)
-      {1, 1},                            // stride (stride_h, stride_w)
-      {1, 1}                             // dilation (dilation_h, dilation_w) 
-    ));
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>(user_size));
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Fprop_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
-  128x128_8x4_64x32x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = float;
-  using ElementB           = float;
-  using ElementC           = float;
-  using ElementAccumulator = float;
-  using ElementCompute     = float;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFprop<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<128, 128, 8>,
-    cutlass::gemm::GemmShape<64, 32, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAdd,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Fprop_Optimized_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
-  32x64_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = float;
-  using ElementB           = float;
-  using ElementC           = float;
-  using ElementAccumulator = float;
-  using ElementCompute     = float;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFprop<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<32, 64, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAdd,
-    cutlass::conv::IteratorAlgorithm::kOptimized
-  >::Kernel;
-
-  using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
-
   /// Run all unit test sizes with device-level Conv2d instance
   EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
 

test/unit/conv/device/conv2d_fprop_implicit_gemm_qf32nhwc_qf32nhwc_qf32nhwc_simt_f32_sm50.cu

@@ -0,0 +1,221 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+/*! \file
+    \brief Tests for device-wide Implicit GEMM interface
+*/
+
+#include "../../common/cutlass_unit_test.h"
+#include "cutlass/cutlass.h"
+
+
+#include "cutlass/conv/kernel/default_conv2d_fprop.h"
+#include "cutlass/conv/device/implicit_gemm_convolution.h"
+
+#include "conv2d_testbed.h"
+
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM50_Device_Conv2d_Fprop_Analytic_ImplicitGemm_qf32nhwc_qf32nhwc_qf32nhwc_simt_f32,
+  16x32_8x2_16x16x8) {
+
+  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
+  using ElementA           = cutlass::Quaternion<float>;
+  using ElementB           = cutlass::Quaternion<float>;
+  using ElementC           = cutlass::Quaternion<float>;
+  using ElementAccumulator = cutlass::Quaternion<float>;
+  using ElementCompute     = cutlass::Quaternion<float>;
+
+
+  /// Device-level Conv2d instance
+  using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFprop<
+    ElementA, 
+    cutlass::layout::TensorNHWC,
+    ElementB, 
+    cutlass::layout::TensorNHWC,
+    ElementC, 
+    cutlass::layout::TensorNHWC,
+    ElementAccumulator,
+    cutlass::arch::OpClassSimt,
+    cutlass::arch::Sm50,
+    cutlass::gemm::GemmShape<16, 32, 8>,
+    cutlass::gemm::GemmShape<16, 16, 8>, 
+    cutlass::gemm::GemmShape<1, 1, 1>,
+    cutlass::epilogue::thread::LinearCombination<
+      ElementC,
+      1,
+      ElementAccumulator,
+      ElementCompute
+    >,
+    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<4>,
+    2,
+    cutlass::arch::OpMultiplyAdd,
+    cutlass::conv::IteratorAlgorithm::kAnalytic
+  >::Kernel;
+
+  using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
+
+  /// Run all unit test sizes with device-level Conv2d instance
+  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
+
+}
+
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM50_Device_Conv2d_Fprop_Analytic_ImplicitGemm_qf32nhwc_qf32nhwc_qf32nhwc_simt_f32,
+  16x64_8x2_8x32x8) {
+
+  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
+  using ElementA           = cutlass::Quaternion<float>;
+  using ElementB           = cutlass::Quaternion<float>;
+  using ElementC           = cutlass::Quaternion<float>;
+  using ElementAccumulator = cutlass::Quaternion<float>;
+  using ElementCompute     = cutlass::Quaternion<float>;
+
+
+  /// Device-level Conv2d instance
+  using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFprop<
+    ElementA, 
+    cutlass::layout::TensorNHWC,
+    ElementB, 
+    cutlass::layout::TensorNHWC,
+    ElementC, 
+    cutlass::layout::TensorNHWC,
+    ElementAccumulator,
+    cutlass::arch::OpClassSimt,
+    cutlass::arch::Sm50,
+    cutlass::gemm::GemmShape<16, 64, 8>,
+    cutlass::gemm::GemmShape<8, 32, 8>, 
+    cutlass::gemm::GemmShape<1, 1, 1>,
+    cutlass::epilogue::thread::LinearCombination<
+      ElementC,
+      1,
+      ElementAccumulator,
+      ElementCompute
+    >,
+    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<4>,
+    2,
+    cutlass::arch::OpMultiplyAdd,
+    cutlass::conv::IteratorAlgorithm::kOptimized
+  >::Kernel;
+
+  using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
+
+  /// Run all unit test sizes with device-level Conv2d instance
+  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
+
+}
+
+////////////////////////////////////////////////////////////////////////////////
+TEST(SM50_Device_Conv2d_Fprop_Analytic_ImplicitGemm_qf32nhwc_qf32nhwc_qf32nhwc_simt_f32,
+  32x32_8x2_16x16x8) {
+
+  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
+  using ElementA           = cutlass::Quaternion<float>;
+  using ElementB           = cutlass::Quaternion<float>;
+  using ElementC           = cutlass::Quaternion<float>;
+  using ElementAccumulator = cutlass::Quaternion<float>;
+  using ElementCompute     = cutlass::Quaternion<float>;
+
+
+  /// Device-level Conv2d instance
+  using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFprop<
+    ElementA, 
+    cutlass::layout::TensorNHWC,
+    ElementB, 
+    cutlass::layout::TensorNHWC,
+    ElementC, 
+    cutlass::layout::TensorNHWC,
+    ElementAccumulator,
+    cutlass::arch::OpClassSimt,
+    cutlass::arch::Sm50,
+    cutlass::gemm::GemmShape<32, 32, 8>,
+    cutlass::gemm::GemmShape<16, 16, 8>, 
+    cutlass::gemm::GemmShape<1, 1, 1>,
+    cutlass::epilogue::thread::LinearCombination<
+      ElementC,
+      1,
+      ElementAccumulator,
+      ElementCompute
+    >,
+    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<4>,
+    2,
+    cutlass::arch::OpMultiplyAdd,
+    cutlass::conv::IteratorAlgorithm::kAnalytic
+  >::Kernel;
+
+  using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
+
+  /// Run all unit test sizes with device-level Conv2d instance
+  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
+
+}
+
+////////////////////////////////////////////////////////////////////////////////
+TEST(SM50_Device_Conv2d_Fprop_Optimized_ImplicitGemm_qf32nhwc_qf32nhwc_qf32nhwc_simt_f32,
+  16x32_8x2_16x16x8) {
+
+  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
+  using ElementA           = cutlass::Quaternion<float>;
+  using ElementB           = cutlass::Quaternion<float>;
+  using ElementC           = cutlass::Quaternion<float>;
+  using ElementAccumulator = cutlass::Quaternion<float>;
+  using ElementCompute     = cutlass::Quaternion<float>;
+
+
+  /// Device-level Conv2d instance
+  using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFprop<
+    ElementA, 
+    cutlass::layout::TensorNHWC,
+    ElementB, 
+    cutlass::layout::TensorNHWC,
+    ElementC, 
+    cutlass::layout::TensorNHWC,
+    ElementAccumulator,
+    cutlass::arch::OpClassSimt,
+    cutlass::arch::Sm50,
+    cutlass::gemm::GemmShape<16, 32, 8>,
+    cutlass::gemm::GemmShape<16, 16, 8>, 
+    cutlass::gemm::GemmShape<1, 1, 1>,
+    cutlass::epilogue::thread::LinearCombination<
+      ElementC,
+      1,
+      ElementAccumulator,
+      ElementCompute
+    >,
+    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<4>,
+    2,
+    cutlass::arch::OpMultiplyAdd,
+    cutlass::conv::IteratorAlgorithm::kOptimized
+  >::Kernel;
+
+  using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
+
+  /// Run all unit test sizes with device-level Conv2d instance
+  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
+
+}
+

test/unit/conv/device/conv2d_fprop_with_broadcast_sm75.cu

@@ -0,0 +1,90 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+/*! \file
+    \brief Tests for device-wide Implicit GEMM interface
+*/
+
+#include "../../common/cutlass_unit_test.h"
+#include "cutlass/cutlass.h"
+
+#include "cutlass/epilogue/thread/linear_combination_bias_elementwise.h"
+#include "cutlass/epilogue/thread/linear_combination_bias_relu.h"
+
+#include "cutlass/conv/kernel/default_conv2d_fprop_with_broadcast.h"
+#include "cutlass/conv/device/implicit_gemm_convolution.h"
+
+#include "conv2d_with_broadcast_testbed.h"
+        
+#if defined(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
+
+TEST(SM75_Device_Conv2d_Fprop_With_Broadcast_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32,
+  128x128_32x2_64x64x32) {
+
+  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
+  using ElementA           = cutlass::half_t;
+  using ElementB           = cutlass::half_t;
+  using ElementC           = cutlass::half_t;
+  using ElementAccumulator = float;
+  using ElementCompute     = float;
+
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasElementwise<
+    cutlass::half_t,
+    float,
+    float,
+    cutlass::half_t,
+    cutlass::half_t,
+    8,
+    cutlass::epilogue::thread::GELU_taylor<float>
+  >;
+
+  /// Device-level Conv2d instance
+  using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFpropWithBroadcast<
+    ElementA, cutlass::layout::TensorNHWC,
+    ElementB, cutlass::layout::TensorNHWC,
+    ElementC, cutlass::layout::TensorNHWC,
+    ElementAccumulator,
+    cutlass::arch::OpClassTensorOp,
+    cutlass::arch::Sm75,
+    cutlass::gemm::GemmShape<128, 128, 32>,
+    cutlass::gemm::GemmShape<64, 64, 32>,
+    cutlass::gemm::GemmShape<16, 8, 8>,
+    EpilogueOutputOp,
+    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+    2,
+    cutlass::arch::OpMultiplyAdd,
+    cutlass::conv::IteratorAlgorithm::kAnalytic
+  >::Kernel;
+
+  using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
+
+  /// Run all unit test sizes with device-level Conv2d instance
+  EXPECT_TRUE(test::conv::device::TestAllConv2dWithBroadcast<Conv2dFprop>());
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+#endif  // CUTLASS_ARCH_MMA_SM75_SUPPORTED
+
+////////////////////////////////////////////////////////////////////////////////

test/unit/conv/device/conv2d_fprop_with_reduction_sm75.cu

@@ -0,0 +1,88 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+/*! \file
+    \brief Tests for device-wide Implicit GEMM interface
+*/
+
+#include "../../common/cutlass_unit_test.h"
+#include "cutlass/cutlass.h"
+
+#include "cutlass/epilogue/thread/linear_combination_with_elementwise.h"
+
+#include "cutlass/conv/kernel/default_conv2d_fprop_with_reduction.h"
+#include "cutlass/conv/device/implicit_gemm_convolution.h"
+
+#include "conv2d_with_reduction_testbed.h"
+        
+#if defined(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
+
+TEST(SM75_Device_Conv2d_Fprop_With_Reduction_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32,
+  128x128_32x2_64x64x32) {
+
+  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
+  using ElementA           = cutlass::half_t;
+  using ElementB           = cutlass::half_t;
+  using ElementC           = cutlass::half_t;
+  using ElementAccumulator = float;
+  using ElementCompute     = float;
+
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationWithElementwise<
+    float,
+    float,
+    cutlass::half_t,
+    cutlass::half_t,
+    8
+  >;
+
+  /// Device-level Conv2d instance
+  using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFpropWithReduction<
+    ElementA, cutlass::layout::TensorNHWC,
+    ElementB, cutlass::layout::TensorNHWC,
+    ElementC, cutlass::layout::TensorNHWC,
+    ElementAccumulator,
+    cutlass::arch::OpClassTensorOp,
+    cutlass::arch::Sm75,
+    cutlass::gemm::GemmShape<128, 128, 32>,
+    cutlass::gemm::GemmShape<64, 64, 32>,
+    cutlass::gemm::GemmShape<16, 8, 8>,
+    EpilogueOutputOp,
+    cutlass::plus<float>,
+    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+    2,
+    cutlass::arch::OpMultiplyAdd,
+    cutlass::conv::IteratorAlgorithm::kAnalytic
+  >::Kernel;
+
+  using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
+
+  /// Run all unit test sizes with device-level Conv2d instance
+  EXPECT_TRUE(test::conv::device::TestAllConv2dWithReduction<Conv2dFprop>());
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+#endif  // CUTLASS_ARCH_MMA_SM75_SUPPORTED
+
+////////////////////////////////////////////////////////////////////////////////

test/unit/conv/device/conv2d_problems.h

@@ -161,7 +161,7 @@ struct TestbedConv2dProblemSizes {
   void initialize_conv2d_default_sizes() {
 
     ////////////////////////////////////////////////////////////////////////////////////////////
-    // Very Small input size (1x8x8xminimum_channel_size), filter size (3x3 - 7x7), stride (1,1)
+    // Small input size x stride (1,1)
     // C < CTA::K and non-multiples of CTA::K. Typical CTA::K = {32, 64}
     ////////////////////////////////////////////////////////////////////////////////////////////
     
@@ -229,6 +229,58 @@ struct TestbedConv2dProblemSizes {
       {1, 1}                             // dilation (dilation_h, dilation_w) 
     ));
 
+    ////////////////////////////////////////////////////////////////////////////////////////////
+    // Small input size x stride (2,2)
+    // C < CTA::K and non-multiples of CTA::K. Typical CTA::K = {32, 64}
+    ////////////////////////////////////////////////////////////////////////////////////////////
+    conv2d_default_sizes.push_back(cutlass::conv::Conv2dProblemSize( 
+      {1, 11, 11, minimum_channel_size},  // input size  (NHWC)
+      {8, 1, 1, minimum_channel_size},    // filter size (KRSC)
+      {0, 0, 0, 0},                       // padding (pad_h, _, pad_w, _)
+      {2, 2},                             // stride (stride_h, stride_w)
+      {1, 1}                              // dilation (dilation_h, dilation_w) 
+    ));
+
+    conv2d_default_sizes.push_back(cutlass::conv::Conv2dProblemSize( 
+      {1, 11, 11, minimum_channel_size},   // input size  (NHWC)
+      {8, 3, 3, minimum_channel_size},     // filter size (KRSC)
+      {1, 1, 1, 1},                        // padding (pad_h, _, pad_w, _)
+      {2, 2},                              // stride (stride_h, stride_w)
+      {1, 1}                               // dilation (dilation_h, dilation_w) 
+    ));
+
+    conv2d_default_sizes.push_back(cutlass::conv::Conv2dProblemSize( 
+      {1, 13, 13, minimum_channel_size},   // input size  (NHWC)
+      {8, 1, 1, minimum_channel_size},     // filter size (KRSC)
+      {1, 1, 1, 1},                        // padding (pad_h, _, pad_w, _)
+      {2, 2},                              // stride (stride_h, stride_w)
+      {1, 1}                               // dilation (dilation_h, dilation_w) 
+    ));
+
+    conv2d_default_sizes.push_back(cutlass::conv::Conv2dProblemSize( 
+      {1, 8, 8, minimum_channel_size},   // input size  (NHWC)
+      {8, 2, 2, minimum_channel_size},   // filter size (KRSC)
+      {1, 1, 1, 1},    // padding (pad_h, _, pad_w, _)
+      {2, 2},          // stride (stride_h, stride_w)
+      {1, 1}           // dilation (dilation_h, dilation_w) 
+    ));
+  
+    conv2d_default_sizes.push_back(cutlass::conv::Conv2dProblemSize( 
+      {1, 5, 5, minimum_channel_size},   // input size  (NHWC)
+      {8, 3, 3, minimum_channel_size},   // filter size (KRSC)
+      {1, 1, 1, 1},    // padding (pad_h, _, pad_w, _)
+      {2, 2},          // stride (stride_h, stride_w)
+      {1, 1}           // dilation (dilation_h, dilation_w) 
+    ));
+  
+    conv2d_default_sizes.push_back(cutlass::conv::Conv2dProblemSize( 
+      {1, 8, 8, 8},   // input size  (NHWC)
+      {8, 3, 3, 8},   // filter size (KRSC)
+      {0, 0, 0, 0},    // padding (pad_h, _, pad_w, _)
+      {2, 2},          // stride (stride_h, stride_w)
+      {1, 1}           // dilation (dilation_h, dilation_w) 
+    ));
+
     ////////////////////////////////////////////////////////////////////////////////////
     // Medium input size (1x16x16x128), filter size (1x1, 2x2, 3x3, 5x5), stride (1, 1) 
     ////////////////////////////////////////////////////////////////////////////////////
@@ -239,7 +291,15 @@ struct TestbedConv2dProblemSizes {
       {1, 1},             // stride (stride_h, stride_w)
       {1, 1}              // dilation (dilation_h, dilation_w) 
     ));
-  
+    
+    conv2d_default_sizes.push_back(cutlass::conv::Conv2dProblemSize(
+      {1, 19, 37, 160},     // input size  (NHWC)
+      {224, 3, 3, 160},     // filter size (KRSC)
+      {1, 1, 1, 1},         // padding (pad_h, _, pad_w, _)
+      {2, 2},               // stride (stride_h, stride_w)
+      {1, 1}                // dilation (dilation_h, dilation_w)
+    ));
+
     conv2d_default_sizes.push_back(cutlass::conv::Conv2dProblemSize(
       {1, 16, 16, 160},   // input size  (NHWC)
       {224, 2, 3, 160},   // filter size (KRSC)
@@ -284,16 +344,8 @@ struct TestbedConv2dProblemSizes {
     ));
 
     ////////////////////////////////////////////////////////////////////////////////////
-    // Medium input size (1x16x16x128), filter size (1x1, 3,x3, 5x5), stride (2, 2)  
-    ////////////////////////////////////////////////////////////////////////////////////
-    conv2d_default_sizes.push_back(cutlass::conv::Conv2dProblemSize(
-      {1, 19, 37, 160},     // input size  (NHWC)
-      {224, 3, 3, 160},     // filter size (KRSC)
-      {1, 1, 1, 1},         // padding (pad_h, _, pad_w, _)
-      {2, 2},               // stride (stride_h, stride_w)
-      {1, 1}                // dilation (dilation_h, dilation_w)
-    ));
-  
+    // Medium input size, filter size (1x1, 3,x3, 5x5, 7x7), stride (2, 2)  
+    //////////////////////////////////////////////////////////////////////////////////// 
     conv2d_default_sizes.push_back(cutlass::conv::Conv2dProblemSize(
       {1, 16, 16, 288},   // input size  (NHWC)
       {160, 5, 5, 288},   // filter size (KRSC)
@@ -302,6 +354,61 @@ struct TestbedConv2dProblemSizes {
       {1, 1}              // dilation (dilation_h, dilation_w)
     ));
 
+    conv2d_default_sizes.push_back(cutlass::conv::Conv2dProblemSize(
+      {1, 55, 55, 256},   // input size (NHWC)
+      {512, 1, 1, 256},   // filter size (KRSC)
+      {0, 0, 0, 0},       // padding (pad_h, _, pad_w, _)
+      {2, 2},             // stride (stride_h, stride_w)
+      {1, 1}              // dilation (dilation_h, dilation_w)
+    ));
+
+    conv2d_default_sizes.push_back(cutlass::conv::Conv2dProblemSize(
+      {1, 80, 80, 32},    // input size (NHWC)
+      {64, 5, 5, 32},     // filter size (KRSC)
+      {2, 2, 2, 2},       // padding (pad_h, _, pad_w, _)
+      {2, 2},             // stride (stride_h, stride_w)
+      {1, 1}              // dilation (dilation_h, dilation_w)
+    ));
+
+    conv2d_default_sizes.push_back(cutlass::conv::Conv2dProblemSize(
+      {1, 224, 224, 8},   // input size (NHWC)
+      {64, 7, 7, 8},      // filter size (KRSC)
+      {3, 3, 3, 3},       // padding (pad_h, _, pad_w, _)
+      {2, 2},             // stride (stride_h, stride_w)
+      {1, 1}              // dilation (dilation_h, dilation_w)
+    ));
+
+    ////////////////////////////////////////////////////////////////////////////////////
+    // Medium input size stride (3, 3), filter (3, 3), non-default padding
+    ////////////////////////////////////////////////////////////////////////////////////
+    conv2d_default_sizes.push_back(cutlass::conv::Conv2dProblemSize(
+      {1, 27, 27, 256},     // input size (NHWC)
+      {512, 3, 3, 256},     // filter size (KRSC)
+      {0, 0, 0, 0},         // padding (pad_h, _, pad_w, _)
+      {3, 3},               // stride (stride_h, stride_w)
+      {1, 1}                // dilation (dilation_h, dilation_w)
+    ));
+    
+    ////////////////////////////////////////////////////////////////////////////////////
+    // Medium input size *mixed* stride (1, 2) and (2, 1), 
+    // filter (3, 3), default padding
+    ////////////////////////////////////////////////////////////////////////////////////
+    conv2d_default_sizes.push_back(cutlass::conv::Conv2dProblemSize(
+      {1, 27, 27, 256},     // input size (NHWC)
+      {512, 3, 3, 256},     // filter size (KRSC)
+      {1, 1, 1, 1},         // padding (pad_h, _, pad_w, _)
+      {1, 2},               // stride (stride_h, stride_w)
+      {1, 1}                // dilation (dilation_h, dilation_w)
+    ));
+
+    conv2d_default_sizes.push_back(cutlass::conv::Conv2dProblemSize(
+      {1, 27, 27, 256},     // input size (NHWC)
+      {512, 3, 3, 256},     // filter size (KRSC)
+      {1, 1, 1, 1},         // padding (pad_h, _, pad_w, _)
+      {2, 1},               // stride (stride_h, stride_w)
+      {1, 1}                // dilation (dilation_h, dilation_w)
+    ));
+
     /////////////////////////////////////////////////////////////////////////////
     // Additional input size 
     /////////////////////////////////////////////////////////////////////////////
@@ -347,15 +454,15 @@ struct TestbedConv2dProblemSizes {
 
 #if CUTLASS_CONV_UNIT_TEST_RIGOROUS_SIZE_ENABLED                  
   conv2d_rigorous_sizes.push_back(cutlass::conv::Conv2dProblemSize(
-    {1, 124, 224, 96},  // input size  (NHWC)
-    {24, 7, 7, 96},     // filter size (KRSC)
-    {1, 229, 129, 32}   // output size (NPQK)
+    {1, 124, 224, 96},    // input size  (NHWC)
+    {24, 7, 7, 96},       // filter size (KRSC)
+    {1, 229, 129, 32}     // output size (NPQK)
   ));
 
   conv2d_rigorous_sizes.push_back(cutlass::conv::Conv2dProblemSize(
-    {1, 233, 35, 48},                     // input size  (NHWC)
-    {24, 7, 5, 48},                       // filter size (KRSC)
-    {1, 233, 35, 24}                     // output size (NPQK)
+    {1, 233, 35, 48},     // input size  (NHWC)
+    {24, 7, 5, 48},       // filter size (KRSC)
+    {1, 233, 35, 24}      // output size (NPQK)
   ));
 
 #endif 

test/unit/conv/device/conv2d_strided_dgrad_implicit_gemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32_sm80.cu

@@ -0,0 +1,187 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+/*! \file
+    \brief Tests for device-wide Implicit GEMM interface
+*/
+
+#include "../../common/cutlass_unit_test.h"
+#include "cutlass/cutlass.h"
+
+#include "cutlass/conv/kernel/default_conv2d_dgrad.h"
+#include "cutlass/conv/device/implicit_gemm_convolution.h"
+
+#include "conv2d_testbed.h"
+
+#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
+
+////////////////////////////////////////////////////////////////////////////////
+//                           Strided Dgrad (Analytic)
+////////////////////////////////////////////////////////////////////////////////
+TEST(SM80_Device_Conv2d_Strided_Dgrad_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32,
+  128x128_32x3_64x64x32) {
+
+  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
+  using ElementA           = cutlass::half_t;
+  using ElementB           = cutlass::half_t;
+  using ElementC           = float;
+  using ElementAccumulator = float;
+  using ElementCompute     = float;
+
+  /// Device-level Conv2d instance
+  using Conv2dDgradKernel = typename cutlass::conv::kernel::DefaultConv2dDgrad<
+    ElementA, cutlass::layout::TensorNHWC,
+    ElementB, cutlass::layout::TensorNHWC,
+    ElementC, cutlass::layout::TensorNHWC,
+    ElementAccumulator,
+    cutlass::arch::OpClassTensorOp,
+    cutlass::arch::Sm80,
+    cutlass::gemm::GemmShape<128, 128, 32>,
+    cutlass::gemm::GemmShape<64, 64, 32>,
+    cutlass::gemm::GemmShape<16, 8, 16>,
+    cutlass::epilogue::thread::LinearCombination<
+      ElementC,
+      128 / cutlass::sizeof_bits<ElementC>::value,
+      ElementAccumulator,
+      ElementCompute
+    >,
+    cutlass::conv::threadblock::StridedDgradIdentityThreadblockSwizzle<>,
+    3,
+    cutlass::arch::OpMultiplyAdd,
+    cutlass::conv::IteratorAlgorithm::kAnalytic,
+    cutlass::conv::StrideSupport::kStrided
+  >::Kernel;
+
+  using Conv2dDgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dDgradKernel>;
+
+
+  test::conv::device::Conv2dProblemVector problem_size_list;
+
+#if 0 // run specific problem size in the unit test first
+    problem_size_list.push_back(cutlass::conv::Conv2dProblemSize(
+      {1, 56, 56, 8},   // input size (NHWC)
+      {8, 1, 1, 8},   // filter size (KRSC)
+      {0, 0, 0, 0},       // padding (pad_h, _, pad_w, _)
+      {2, 2},             // stride (stride_h, stride_w)
+      {1, 1}              // dilation (dilation_h, dilation_w)
+    ));
+
+    problem_size_list.push_back(cutlass::conv::Conv2dProblemSize(
+      {1, 55, 55, 8},   // input size (NHWC)
+      {8, 1, 1, 8},   // filter size (KRSC)
+      {0, 0, 0, 0},       // padding (pad_h, _, pad_w, _)
+      {2, 2},             // stride (stride_h, stride_w)
+      {1, 1}              // dilation (dilation_h, dilation_w)
+    ));
+
+#endif
+
+  /// Run all unit test sizes with device-level Conv2d instance
+  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dDgrad>(problem_size_list));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+TEST(SM80_Device_Conv2d_Strided_Dgrad_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32,
+  128x256_32x3_64x64x32) {
+
+  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
+  using ElementA           = cutlass::half_t;
+  using ElementB           = cutlass::half_t;
+  using ElementC           = float;
+  using ElementAccumulator = float;
+  using ElementCompute     = float;
+
+  /// Device-level Conv2d instance
+  using Conv2dDgradKernel = typename cutlass::conv::kernel::DefaultConv2dDgrad<
+    ElementA, cutlass::layout::TensorNHWC,
+    ElementB, cutlass::layout::TensorNHWC,
+    ElementC, cutlass::layout::TensorNHWC,
+    ElementAccumulator,
+    cutlass::arch::OpClassTensorOp,
+    cutlass::arch::Sm80,
+    cutlass::gemm::GemmShape<128, 256, 32>,
+    cutlass::gemm::GemmShape<64, 64, 32>,
+    cutlass::gemm::GemmShape<16, 8, 16>,
+    cutlass::epilogue::thread::LinearCombination<
+      ElementC,
+      128 / cutlass::sizeof_bits<ElementC>::value,
+      ElementAccumulator,
+      ElementCompute
+    >,
+    cutlass::conv::threadblock::StridedDgradIdentityThreadblockSwizzle<>,
+    3,
+    cutlass::arch::OpMultiplyAdd,
+    cutlass::conv::IteratorAlgorithm::kAnalytic,
+    cutlass::conv::StrideSupport::kStrided
+  >::Kernel;
+
+  using Conv2dDgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dDgradKernel>;
+
+  /// Run all unit test sizes with device-level Conv2d instance
+  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dDgrad>());
+}
+
+////////////////////////////////////////////////////////////////////////////////
+TEST(SM80_Device_Conv2d_Strided_Dgrad_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32,
+  128x256_64x3_64x64x64) {
+
+  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
+  using ElementA           = cutlass::half_t;
+  using ElementB           = cutlass::half_t;
+  using ElementC           = float;
+  using ElementAccumulator = float;
+  using ElementCompute     = float;
+
+  /// Device-level Conv2d instance
+  using Conv2dDgradKernel = typename cutlass::conv::kernel::DefaultConv2dDgrad<
+    ElementA, cutlass::layout::TensorNHWC,
+    ElementB, cutlass::layout::TensorNHWC,
+    ElementC, cutlass::layout::TensorNHWC,
+    ElementAccumulator,
+    cutlass::arch::OpClassTensorOp,
+    cutlass::arch::Sm80,
+    cutlass::gemm::GemmShape<128, 256, 64>,
+    cutlass::gemm::GemmShape<64, 64, 64>,
+    cutlass::gemm::GemmShape<16, 8, 16>,
+    cutlass::epilogue::thread::LinearCombination<
+      ElementC,
+      128 / cutlass::sizeof_bits<ElementC>::value,
+      ElementAccumulator,
+      ElementCompute
+    >,
+    cutlass::conv::threadblock::StridedDgradIdentityThreadblockSwizzle<>,
+    3,
+    cutlass::arch::OpMultiplyAdd,
+    cutlass::conv::IteratorAlgorithm::kAnalytic,
+    cutlass::conv::StrideSupport::kStrided
+  >::Kernel;
+
+  using Conv2dDgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dDgradKernel>;
+
+  /// Run all unit test sizes with device-level Conv2d instance
+  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dDgrad>());
+}
+////////////////////////////////////////////////////////////////////////////////
+
+#endif  // CUTLASS_ARCH_MMA_SM80_SUPPORTED

test/unit/conv/device/conv2d_testbed.h

@@ -81,7 +81,7 @@ public:
   >;
 
   using ReductionDevice = cutlass::reduction::device::ReduceSplitK<ReductionKernel>;
-
+  using ReductionStrideIndex = typename ReductionDevice::StrideIndex;
 
 public:
 
@@ -161,7 +161,7 @@ public:
     initialize_tensor(tensor_A.host_view(), init_A, seed); 
     initialize_tensor(tensor_B.host_view(), init_B, seed * 17); 
     initialize_tensor(tensor_C.host_view(), init_C, seed * 39);
-
+    
     tensor_A.sync_device();
     tensor_B.sync_device();
     tensor_C.sync_device();
@@ -214,7 +214,7 @@ public:
 
 #if 0 //display conv2d problem size for debugging
     std::cout << problem_size << std::endl
-              << "alpha, beta: (" << float(alpha) << ", " << float(beta) << ")" << std::endl
+              << "alpha, beta: (" << alpha << ", " << beta << ")" << std::endl
               << "split_k_mode: " << ((split_k_mode == cutlass::conv::SplitKMode::kSerial) ? "(serial)" : "(parallel)") << std::endl
               << std::endl;
 #endif
@@ -262,7 +262,7 @@ public:
     if (status != cutlass::Status::kSuccess) {
       return false;
     }
-  
+
     // run conv2d operator
     status = conv2d_op();
     
@@ -271,6 +271,7 @@ public:
       return false;
     }
 
+
     if (split_k_mode == cutlass::conv::SplitKMode::kParallel) {
 
       // configure parallel reduction operator 
@@ -280,10 +281,20 @@ public:
         cutlass::conv::implicit_gemm_problem_size(kConvolutionalOperator, problem_size).mn(),
         problem_size.split_k_slices,
         cutlass::conv::implicit_gemm_tensor_c_size(kConvolutionalOperator, problem_size),
-        {reinterpret_cast<ElementAccumulator*> (workspace.get()), tensor_C.stride(Conv2d::ImplicitGemmKernel::kTensorCStrideIdx)},
-        {tensor_D_computed.device_data(), tensor_C.stride(Conv2d::ImplicitGemmKernel::kTensorCStrideIdx)},
-        {tensor_C.device_data(), tensor_C.stride(Conv2d::ImplicitGemmKernel::kTensorCStrideIdx)},
-        {alpha, beta} // apply alpha, beta to obtain the following equation alpha * ReduceAdd(A * B) + beta * C 
+        {
+          reinterpret_cast<ElementAccumulator*> (workspace.get()),
+          ReductionStrideIndex(tensor_C.stride()[Conv2d::ImplicitGemmKernel::kTensorCStrideIdx])
+        },
+        {
+          tensor_D_computed.device_data(),
+          ReductionStrideIndex(tensor_C.stride()[Conv2d::ImplicitGemmKernel::kTensorCStrideIdx])
+        },
+        {
+          tensor_C.device_data(),
+          ReductionStrideIndex(tensor_C.stride()[Conv2d::ImplicitGemmKernel::kTensorCStrideIdx])
+        },
+        // apply alpha, beta to obtain the following equation alpha * ReduceAdd(A * B) + beta * C 
+        {alpha, beta} 
       );
 
       status = reduction_op.initialize(reduction_args, nullptr);
@@ -302,7 +313,11 @@ public:
       }
     }
     bool passed = false;
-    
+
+    cudaError_t result = cudaDeviceSynchronize();
+    EXPECT_EQ(result, cudaSuccess) << " device reference error: " 
+                                   << cudaGetErrorString(result);
+
     tensor_D_computed.sync_host();
 
 #if CUTLASS_CONV_TEST_UNIT_REFERENCE_DEVICE_ENABLED
@@ -326,10 +341,6 @@ public:
       alpha, 
       beta);
 
-    cudaError_t result = cudaDeviceSynchronize();
-    EXPECT_EQ(result, cudaSuccess) << " device reference error: " 
-                                   << cudaGetErrorString(result);
-
     // sync host (copy device data to host) for dumping error output in case of mismatches
     tensor_D_reference.sync_host();
     
@@ -445,7 +456,7 @@ bool TestAllConv2d(
   Conv2dProblemVector const *problem_vectors[] = {
     &conv_test_sizes,                               // run user specified sizes
     &conv_problems.conv2d_default_sizes,            // run default and cudnn bug sizes
-    &conv_problems.conv2d_resnet50_sizes,           // run resnet50 sizes
+    //&conv_problems.conv2d_resnet50_sizes,           // run resnet50 sizes
 #if CUTLASS_CONV_UNIT_TEST_RIGOROUS_SIZE_ENABLED 
     &conv_problems.conv2d_rigorous_sizes,           // run large and rigorous sizes if enabled
 #endif
@@ -467,7 +478,7 @@ bool TestAllConv2d(
       // Procedurally disable certain cases
       //
   
-      // CUTLASS DGRAD's unity stride specialization only support stride {1, 1} 
+      // CUTLASS DGRAD's *unity* stride specialization only support stride {1, 1} 
       if ((ImplicitGemm::kConvolutionalOperator == 
             cutlass::conv::Operator::kDgrad) && 
           (ImplicitGemm::ImplicitGemmKernel::Mma::IteratorA::kStrideSupport == 
@@ -477,6 +488,18 @@ bool TestAllConv2d(
         }
       }
 
+      // CUTLASS DGRAD's *strided* stride specialization supports all stride {stride_h, stride_w} 
+      // Although strided dgrad works for all stride combinations, we are only going 
+      // to run strided dgrad for non-unity strides 
+      if ((ImplicitGemm::kConvolutionalOperator == 
+            cutlass::conv::Operator::kDgrad) && 
+          (ImplicitGemm::ImplicitGemmKernel::Mma::IteratorA::kStrideSupport == 
+            cutlass::conv::StrideSupport::kStrided)) {
+         if (((conv_problem.stride_h == 1) && (conv_problem.stride_w == 1))) {
+           continue;
+         }
+      }
+      
       //
       // Test
       //
@@ -491,7 +514,7 @@ bool TestAllConv2d(
       if (!passed) {
         return false;
       }
-
+      
       // test mode = convolution
       passed = testbed.run(
         conv_problem.reset_mode(cutlass::conv::Mode::kConvolution),
@@ -503,6 +526,30 @@ bool TestAllConv2d(
     }
   }
 
+  // CUTLASS DGRAD's *strided* specialization does not support split-k mode 
+  if ((ImplicitGemm::kConvolutionalOperator == 
+          cutlass::conv::Operator::kDgrad) && 
+      (ImplicitGemm::ImplicitGemmKernel::Mma::IteratorA::kStrideSupport == 
+        cutlass::conv::StrideSupport::kStrided)) {
+
+    passed = testbed.run(
+      cutlass::conv::Conv2dProblemSize(
+      {1, 56, 56, 8},   // input size (NHWC)
+      {8, 1, 1, 8},     // filter size (KRSC)
+      {0, 0, 0, 0},     // padding (pad_h, _, pad_w, _)
+      {2, 2},           // stride (stride_h, stride_w)
+      {1, 1}),          // dilation (dilation_h, dilation_w)
+      cutlass::conv::SplitKMode::kSerial,
+      cutlass::from_real<typename ImplicitGemm::ElementCompute>(2.0), 
+      cutlass::from_real<typename ImplicitGemm::ElementCompute>(2.0));
+
+    if (!passed) {
+      return false;
+    }
+
+    return passed;
+  }
+
   // Sweep split-k-slice using serial and prallel reduction with non-unity alpha and non-zero beta for 
   // a single conv2d problem size. Convolution unit tests take a long time to run so only sweep parameters 
   // which are abolutely neccessary to catch functional bugs. The below code does provide option to sweep 

test/unit/conv/device/conv2d_testbed_interleaved.h

@@ -82,7 +82,7 @@ public:
   >;
 
   using ReductionDevice = cutlass::reduction::device::ReduceSplitK<ReductionKernel>;
-
+  using ReductionStrideIndex = typename ReductionDevice::StrideIndex;
 
 public:
 
@@ -245,10 +245,20 @@ public:
         cutlass::conv::implicit_gemm_problem_size(kConvolutionalOperator, problem_size).mn(),
         problem_size.split_k_slices,
         cutlass::conv::implicit_gemm_tensor_c_size(kConvolutionalOperator, problem_size),
-        {reinterpret_cast<ElementAccumulator*> (workspace.get()), tensor_C.stride(Conv2d::ImplicitGemmKernel::kTensorCStrideIdx)},
-        {tensor_D_computed.device_data(), tensor_C.stride(Conv2d::ImplicitGemmKernel::kTensorCStrideIdx)},
-        {tensor_C.device_data(), tensor_C.stride(Conv2d::ImplicitGemmKernel::kTensorCStrideIdx)},
-        {alpha, beta} // apply alpha, beta to obtain the following equation alpha * ReduceAdd(A * B) + beta * C 
+        {
+          reinterpret_cast<ElementAccumulator*> (workspace.get()),
+          ReductionStrideIndex(tensor_C.stride()[Conv2d::ImplicitGemmKernel::kTensorCStrideIdx])
+        },
+        {
+          tensor_D_computed.device_data(),
+          ReductionStrideIndex(tensor_C.stride()[Conv2d::ImplicitGemmKernel::kTensorCStrideIdx])
+        },
+        {
+          tensor_C.device_data(),
+          ReductionStrideIndex(tensor_C.stride()[Conv2d::ImplicitGemmKernel::kTensorCStrideIdx])
+        },
+        // apply alpha, beta to obtain the following equation alpha * ReduceAdd(A * B) + beta * C 
+        {alpha, beta}
       );
 
       status = reduction_op.initialize(reduction_args, nullptr);

test/unit/conv/device/conv2d_wgrad_implicit_gemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32_sm50.cu

@@ -36,51 +36,6 @@
 #include "conv2d_testbed.h"
 
 
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM50_Device_Conv2d_Wgrad_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
-  32x64_8x2_32x32x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::complex<float>;
-  using ElementB           = cutlass::complex<float>;
-  using ElementC           = cutlass::complex<float>;
-  using ElementAccumulator = cutlass::complex<float>;
-  using ElementCompute     = cutlass::complex<float>;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dWgradKernel = typename cutlass::conv::kernel::DefaultConv2dWgrad<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm50,
-    cutlass::gemm::GemmShape<32, 64, 8>,
-    cutlass::gemm::GemmShape<32, 32, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    2,
-    cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dWgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dWgradKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dWgrad>());
-
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 TEST(SM50_Device_Conv2d_Wgrad_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
   64x64_8x2_32x32x8) {

test/unit/conv/device/conv2d_wgrad_implicit_gemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32_sm80.cu

@@ -37,95 +37,6 @@
 
 #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
 
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Wgrad_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
-  32x64_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::complex<float>;
-  using ElementB           = cutlass::complex<float>;
-  using ElementC           = cutlass::complex<float>;
-  using ElementAccumulator = cutlass::complex<float>;
-  using ElementCompute     = cutlass::complex<float>;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dWgradKernel = typename cutlass::conv::kernel::DefaultConv2dWgrad<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<32, 64, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dWgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dWgradKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dWgrad>());
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Wgrad_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
-  64x64_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::complex<float>;
-  using ElementB           = cutlass::complex<float>;
-  using ElementC           = cutlass::complex<float>;
-  using ElementAccumulator = cutlass::complex<float>;
-  using ElementCompute     = cutlass::complex<float>;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dWgradKernel = typename cutlass::conv::kernel::DefaultConv2dWgrad<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<64, 64, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dWgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dWgradKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dWgrad>());
-
-}
 
 ////////////////////////////////////////////////////////////////////////////////
 TEST(SM80_Device_Conv2d_Wgrad_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
@@ -172,96 +83,6 @@ TEST(SM80_Device_Conv2d_Wgrad_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_s
 
 }
 
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Wgrad_Analytic_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
-  128x128_8x4_64x32x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::complex<float>;
-  using ElementB           = cutlass::complex<float>;
-  using ElementC           = cutlass::complex<float>;
-  using ElementAccumulator = cutlass::complex<float>;
-  using ElementCompute     = cutlass::complex<float>;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dWgradKernel = typename cutlass::conv::kernel::DefaultConv2dWgrad<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<128, 128, 8>,
-    cutlass::gemm::GemmShape<64, 32, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dWgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dWgradKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dWgrad>());
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Wgrad_Optimized_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
-  32x64_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = cutlass::complex<float>;
-  using ElementB           = cutlass::complex<float>;
-  using ElementC           = cutlass::complex<float>;
-  using ElementAccumulator = cutlass::complex<float>;
-  using ElementCompute     = cutlass::complex<float>;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dWgradKernel = typename cutlass::conv::kernel::DefaultConv2dWgrad<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<32, 64, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAddComplex,
-    cutlass::conv::IteratorAlgorithm::kOptimized
-  >::Kernel;
-
-  using Conv2dWgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dWgradKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dWgrad>());
-
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 TEST(SM80_Device_Conv2d_Wgrad_Optimized_ImplicitGemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32,
   128x128_8x4_64x32x8) {

test/unit/conv/device/conv2d_wgrad_implicit_gemm_f32nhwc_f32nhwc_f32nhwc_simt_f32_sm80.cu

@@ -37,151 +37,6 @@
 
 #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
 
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Wgrad_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
-  32x64_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = float;
-  using ElementB           = float;
-  using ElementC           = float;
-  using ElementAccumulator = float;
-  using ElementCompute     = float;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dWgradKernel = typename cutlass::conv::kernel::DefaultConv2dWgrad<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<32, 64, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAdd,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dWgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dWgradKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dWgrad>());
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Wgrad_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
-  64x64_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = float;
-  using ElementB           = float;
-  using ElementC           = float;
-  using ElementAccumulator = float;
-  using ElementCompute     = float;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dWgradKernel = typename cutlass::conv::kernel::DefaultConv2dWgrad<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<64, 64, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAdd,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dWgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dWgradKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dWgrad>());
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Wgrad_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
-  128x128_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = float;
-  using ElementB           = float;
-  using ElementC           = float;
-  using ElementAccumulator = float;
-  using ElementCompute     = float;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dWgradKernel = typename cutlass::conv::kernel::DefaultConv2dWgrad<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<128, 128, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAdd,
-    cutlass::conv::IteratorAlgorithm::kAnalytic
-  >::Kernel;
-
-  using Conv2dWgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dWgradKernel>;
-
-  test::conv::device::Conv2dProblemVector user_size;
-
-  user_size.push_back(cutlass::conv::Conv2dProblemSize(
-      {1, 8, 8, 4},   // input size  (NHWC)
-      {8, 1, 1, 4},   // filter size (KRSC)
-      {0, 0, 0, 0},                      // padding (pad_h, _, pad_w, _)
-      {1, 1},                            // stride (stride_h, stride_w)
-      {1, 1}                             // dilation (dilation_h, dilation_w) 
-    ));
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dWgrad>(user_size));
-
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 TEST(SM80_Device_Conv2d_Wgrad_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
   128x128_8x4_64x32x8) {
@@ -227,51 +82,6 @@ TEST(SM80_Device_Conv2d_Wgrad_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt
 
 }
 
-////////////////////////////////////////////////////////////////////////////////
-TEST(SM80_Device_Conv2d_Wgrad_Optimized_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
-  32x64_8x4_32x64x8) {
-
-  /// Conv operation element types for the Gemm equivalent (ImplicitGemm)
-  using ElementA           = float;
-  using ElementB           = float;
-  using ElementC           = float;
-  using ElementAccumulator = float;
-  using ElementCompute     = float;
-
-
-  /// Device-level Conv2d instance
-  using Conv2dWgradKernel = typename cutlass::conv::kernel::DefaultConv2dWgrad<
-    ElementA, 
-    cutlass::layout::TensorNHWC,
-    ElementB, 
-    cutlass::layout::TensorNHWC,
-    ElementC, 
-    cutlass::layout::TensorNHWC,
-    ElementAccumulator,
-    cutlass::arch::OpClassSimt,
-    cutlass::arch::Sm80,
-    cutlass::gemm::GemmShape<32, 64, 8>,
-    cutlass::gemm::GemmShape<32, 64, 8>, 
-    cutlass::gemm::GemmShape<1, 1, 1>,
-    cutlass::epilogue::thread::LinearCombination<
-      ElementC,
-      1,
-      ElementAccumulator,
-      ElementCompute
-    >,
-    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-    4,
-    cutlass::arch::OpMultiplyAdd,
-    cutlass::conv::IteratorAlgorithm::kOptimized
-  >::Kernel;
-
-  using Conv2dWgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dWgradKernel>;
-
-  /// Run all unit test sizes with device-level Conv2d instance
-  EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dWgrad>());
-
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 TEST(SM80_Device_Conv2d_Wgrad_Optimized_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
   128x128_8x4_64x32x8) {

test/unit/conv/device/conv2d_with_broadcast_testbed.h

@@ -0,0 +1,551 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+/*! \file
+    \brief Implicit GEMM testbed
+*/
+#pragma once
+
+#include "../../common/cutlass_unit_test.h"
+#include "cutlass/cutlass.h"
+
+#include "cutlass/conv/device/implicit_gemm_convolution.h"
+#include "cutlass/reduction/device/reduce_split_k.h"
+#include "cutlass/reduction/thread/reduction_operators.h"
+
+#include "conv2d_problems.h"
+
+#include "cutlass/util/host_tensor.h"
+#include "cutlass/util/reference/host/tensor_fill.h"
+#include "cutlass/util/reference/device/tensor_compare.h"
+#include "cutlass/util/reference/host/tensor_compare.h"
+
+#include "cutlass/util/reference/host/convolution.h"
+#include "cutlass/util/reference/device/convolution.h"
+
+#include "cutlass/core_io.h"
+#include "cutlass/util/tensor_view_io.h"
+
+namespace test {
+namespace conv {
+namespace device {
+
+template <typename Conv2d>
+class TestbedConv2dWithBroadcast {
+public:
+
+  using ElementA = typename Conv2d::ElementA;
+  using LayoutA = typename Conv2d::LayoutA;
+  using ElementB = typename Conv2d::ElementB;
+  using LayoutB = typename Conv2d::LayoutB;
+  using ElementC = typename Conv2d::ElementC;
+  using LayoutC = typename Conv2d::LayoutC;
+  using ElementAccumulator = typename Conv2d::ElementAccumulator;
+  using ElementCompute = typename Conv2d::ElementCompute;
+  using EpilogueOutputOp = typename Conv2d::EpilogueOutputOp;
+
+  static cutlass::conv::Operator const kConvolutionalOperator = Conv2d::kConvolutionalOperator;
+
+public:
+
+  /// Initialization
+  cutlass::Distribution::Kind init_A;
+  cutlass::Distribution::Kind init_B;
+  cutlass::Distribution::Kind init_C;
+  uint64_t seed;
+
+  cutlass::HostTensor<ElementA, LayoutA> tensor_A;
+  cutlass::HostTensor<ElementB, LayoutB> tensor_B;
+  cutlass::HostTensor<ElementC, LayoutC> tensor_C;
+  cutlass::HostTensor<ElementC, LayoutC> tensor_D_computed;
+  cutlass::HostTensor<ElementC, LayoutC> tensor_D_reference;
+
+public:
+
+  TestbedConv2dWithBroadcast(
+    cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
+    cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
+    cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
+    uint64_t seed_ = 2080
+  ):
+    init_A(init_A_), init_B(init_B_), init_C(init_C_), seed(seed_) {
+
+  }
+
+    /// Helper to initialize a tensor view
+  template <typename Element, typename Layout>
+  void initialize_tensor(
+    cutlass::TensorView<Element, Layout> view, 
+    cutlass::Distribution::Kind dist_kind,
+    uint64_t seed) {
+
+    if (dist_kind == cutlass::Distribution::Uniform) {
+
+      int scope;
+      int bits = cutlass::sizeof_bits<Element>::value;
+
+      if (bits <= 8) {
+        scope = 2;
+      }
+      else if (bits == 16) {
+        scope = 3;
+      }
+      else {
+        scope = 8;
+      }
+      cutlass::reference::host::TensorFillRandomUniform(
+        view, seed, scope, -scope, 0);
+    } 
+    else if (dist_kind == cutlass::Distribution::Identity) {
+
+      cutlass::reference::host::TensorFillIdentity(view);
+    } 
+    else if (dist_kind == cutlass::Distribution::Gaussian) {
+
+      cutlass::reference::host::TensorFillRandomGaussian(view, seed, 0, 0.5);
+    }
+    else if (dist_kind == cutlass::Distribution::Sequential) {
+
+      cutlass::reference::host::BlockFillSequential(view.data(), view.capacity());
+    } 
+    else {
+    }
+  }
+
+  void initialize(
+    cutlass::conv::Conv2dProblemSize const &problem_size, uint64_t seed = 2019) {
+        
+    tensor_A.resize(implicit_gemm_tensor_a_extent(kConvolutionalOperator, problem_size));
+    tensor_B.resize(implicit_gemm_tensor_b_extent(kConvolutionalOperator, problem_size));
+    tensor_C.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
+    tensor_D_computed.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
+    tensor_D_reference.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
+
+    initialize_tensor(tensor_A.host_view(), init_A, seed); 
+    initialize_tensor(tensor_B.host_view(), init_B, seed * 17); 
+    initialize_tensor(tensor_C.host_view(), init_C, seed * 39);
+    
+    tensor_A.sync_device();
+    tensor_B.sync_device();
+    tensor_C.sync_device();
+    tensor_D_computed.sync_device();
+    tensor_D_reference.sync_device();
+  }
+
+  bool sufficient() const {
+    //
+    // Determine SMEM requirements and waive if not satisfied
+    //
+
+    int smem_size = int(sizeof(typename Conv2d::ImplicitGemmKernel::SharedStorage));
+
+    cudaDeviceProp properties;
+    int device_idx;
+    cudaError_t result = cudaGetDevice(&device_idx);
+
+    if (result != cudaSuccess) {
+      throw std::runtime_error("cudaGetDevice() API call failed.");
+    }
+
+    result = cudaGetDeviceProperties(&properties, device_idx);
+
+    if (result != cudaSuccess) {
+      throw std::runtime_error("cudaGetDeviceProperties() failed");
+    }
+
+    if (properties.sharedMemPerMultiprocessor < smem_size) {
+      return false;
+    }
+
+    return true;
+  }
+
+  /// Executes one test
+  bool run(
+    cutlass::conv::Conv2dProblemSize const &problem_size,
+    cutlass::conv::SplitKMode const &split_k_mode = cutlass::conv::SplitKMode::kSerial,
+    ElementCompute alpha = ElementCompute(1),
+    ElementCompute beta = ElementCompute(0)) {
+
+    // Waive test if insufficient CUDA device
+    if (!sufficient()) {
+      if (CUTLASS_TEST_UNIT_ENABLE_WARNINGS) {
+        std::cerr << "Test waived due to insufficient CUDA device." << std::endl;
+      }
+      return true;
+    }
+
+#if 0 //display conv2d problem size for debugging
+    std::cout << problem_size << std::endl
+              << "alpha, beta: (" << alpha << ", " << beta << ")" << std::endl
+              << "split_k_mode: " << ((split_k_mode == cutlass::conv::SplitKMode::kSerial) ? "(serial)" : "(parallel)") << std::endl
+              << std::endl;
+#endif
+
+    initialize(problem_size);
+
+    // configure the operator
+    Conv2d conv2d_op;
+
+    typename Conv2d::Arguments conv2d_args(
+      problem_size,
+      tensor_A.device_ref(),
+      tensor_B.device_ref(),
+      tensor_C.device_ref(),
+      tensor_D_computed.device_ref(),
+      {alpha, beta},
+      split_k_mode
+    );
+
+    // find workspace requirement for parallel split-k reduction
+    size_t workspace_size = Conv2d::get_workspace_size(conv2d_args);
+
+    cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
+
+    cutlass::Status status = conv2d_op.initialize(conv2d_args, workspace.get());
+
+    if (status != cutlass::Status::kSuccess) {
+      cudaError_t error = cudaGetLastError();
+      std::cerr << "This test is not supported: " << cudaGetErrorString(error) << "\n";
+      return true;
+    }
+
+    // conv2d operation with parallel split-k-mode
+    if (split_k_mode == cutlass::conv::SplitKMode::kParallel) {
+
+      // conv2d output is written to workspace in global memory
+      conv2d_args.ref_D.reset(reinterpret_cast<ElementC*>(workspace.get()));
+      // accumulate mma for each cta in k-dimension (1.0 * A * B)
+      conv2d_args.output_op = {ElementCompute(1), ElementCompute(0)}; 
+      // update conv2d operator arguments
+      status = conv2d_op.update(conv2d_args, workspace.get());
+    }
+    
+    EXPECT_TRUE(status == cutlass::Status::kSuccess);
+    if (status != cutlass::Status::kSuccess) {
+      return false;
+    }
+
+    // run conv2d operator
+    status = conv2d_op();
+    
+    EXPECT_TRUE(status == cutlass::Status::kSuccess);
+    if (status != cutlass::Status::kSuccess) {
+      return false;
+    }
+
+    bool passed = false;
+
+    cudaError_t result = cudaDeviceSynchronize();
+    EXPECT_EQ(result, cudaSuccess) << " device reference error: " 
+                                   << cudaGetErrorString(result);
+
+    tensor_D_computed.sync_host();
+
+#if CUTLASS_CONV_TEST_UNIT_REFERENCE_DEVICE_ENABLED
+
+    cutlass::reference::device::Conv2d<
+      ElementA,
+      LayoutA,
+      ElementB,
+      LayoutB,
+      ElementC,
+      LayoutC,
+      ElementCompute,
+      ElementAccumulator 
+    >(
+      kConvolutionalOperator,
+      problem_size,
+      tensor_A.device_ref(),
+      tensor_B.device_ref(),
+      tensor_C.device_ref(),
+      tensor_D_reference.device_ref(),
+      alpha, 
+      beta);
+
+    // sync host (copy device data to host) for dumping error output in case of mismatches
+    tensor_D_reference.sync_host();
+    
+#else 
+
+    cutlass::reference::host::Conv2d<
+      ElementA,
+      LayoutA,
+      ElementB,
+      LayoutB,
+      ElementC,
+      LayoutC,
+      ElementCompute,
+      ElementAccumulator
+    >(
+      kConvolutionalOperator,
+      problem_size,
+      tensor_A.host_ref(),
+      tensor_B.host_ref(),
+      tensor_C.host_ref(),
+      tensor_D_reference.host_ref(),
+      alpha, 
+      beta);
+
+#endif
+    passed = cutlass::reference::host::TensorEquals(
+      tensor_D_computed.host_view(), 
+      tensor_D_reference.host_view());
+
+    EXPECT_TRUE(passed);
+
+    if (!passed) {
+      std::stringstream fname;
+
+      fname << "error_Conv2d_ImplicitGemm_device_"
+        << (split_k_mode == cutlass::conv::SplitKMode::kSerial ? "serial_reduction_" : "parallel_reduction_")
+        << (Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kFprop ? "fprop_" :
+            (Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ? "dgrad_" : "wgrad_")) 
+        << "nhwc_"
+        << problem_size.N << "x"
+        << problem_size.H << "x"
+        << problem_size.W << "x"
+        << problem_size.C 
+        << "_krsc_"
+        << problem_size.K << "x"
+        << problem_size.R << "x"
+        << problem_size.S << "x"
+        << problem_size.C 
+        << "_padding_" 
+        << problem_size.pad_h << "x"
+        << problem_size.pad_w 
+        << "_stride_"  
+        << problem_size.stride_h << "x"
+        << problem_size.stride_w 
+        << "_dilation_"
+        << problem_size.dilation_h << "x"
+        << problem_size.dilation_w << "_"
+        << (problem_size.mode == cutlass::conv::Mode::kCrossCorrelation ? "xcorr_" : "conv_")
+        << Conv2d::ThreadblockShape::kM << "x"  
+        << Conv2d::ThreadblockShape::kN << "x"  
+        << Conv2d::ThreadblockShape::kK << "_"
+        << Conv2d::WarpShape::kM << "x"  
+        << Conv2d::WarpShape::kN << "x"  
+        << Conv2d::WarpShape::kK << ".txt";
+
+      std::cout << fname.str() << std::endl;
+
+      std::ofstream results(fname.str());
+
+      results << problem_size << std::endl;
+
+      results
+        << "\nA:\n" << tensor_A.host_view() << "\n"
+        << "\nB:\n" << tensor_B.host_view() << "\n"
+        << "\nC:\n" << tensor_C.host_view() << "\n"
+        << "\nD reference:\n" << tensor_D_reference.host_view() << "\n"
+        << "\nD computed:\n" << tensor_D_computed.host_view() << "\n";
+
+    }
+
+    return passed;
+  }
+
+};
+
+/////////////////////////////////////////////////////////////////////////////////////////////////////////
+// TestAllConv: Runs cutlass::conv::device::ImplicitGemmConvolution operator and compares it with reference
+// TestAllConv runs conv operator on default conv problem sizes from test::conv::device::TestbedConv2dProblemSizes
+// Additionaly, each conv2d test can provide conv problem sizes (conv_test_sizes) and blacklist of sizes 
+// (conv_blacklist_sizes)
+/////////////////////////////////////////////////////////////////////////////////////////////////////////////
+template <typename ImplicitGemm>
+bool TestAllConv2dWithBroadcast(
+  const Conv2dProblemVector & conv_test_sizes = Conv2dProblemVector(),
+  const Conv2dProblemVector & conv_blacklist_sizes = Conv2dProblemVector()) {
+
+  bool passed = true;
+
+  //
+  // Testbed object
+  //
+
+  TestbedConv2dWithBroadcast<ImplicitGemm> testbed;
+
+  //
+  // Get conv problem sizes to run conv operator 
+  //
+  TestbedConv2dProblemSizes conv_problems(128/cutlass::sizeof_bits<typename ImplicitGemm::ElementA>::value);
+
+  // Vector of conv2d problem sizes to avoid duplicate runs
+  Conv2dProblemVector conv_tested_sizes;
+
+  Conv2dProblemVector const *problem_vectors[] = {
+    &conv_test_sizes,                               // run user specified sizes
+    &conv_problems.conv2d_default_sizes,            // run default and cudnn bug sizes
+    &conv_problems.conv2d_resnet50_sizes,           // run resnet50 sizes
+#if CUTLASS_CONV_UNIT_TEST_RIGOROUS_SIZE_ENABLED 
+    &conv_problems.conv2d_rigorous_sizes,           // run large and rigorous sizes if enabled
+#endif
+  };
+
+  // Sweep conv2d problem sizes (split-k-mode=kSerial, split-k-slice=1, alpha=1.0, beta=0.0)
+  for (Conv2dProblemVector const * problem_vector : problem_vectors) {
+
+    //  Run conv testbed on default convolution sizes
+    for(auto conv_problem : *problem_vector) {
+
+      // Skip blacklist and avoid duplicate problem sizes
+      if (std::find(conv_blacklist_sizes.begin(), conv_blacklist_sizes.end(), conv_problem) != conv_blacklist_sizes.end() ||
+          std::find(conv_tested_sizes.begin(), conv_tested_sizes.end(), conv_problem) != conv_tested_sizes.end()) {
+        continue;
+      }
+
+      //
+      // Procedurally disable certain cases
+      //
+  
+      // CUTLASS DGRAD's *unity* stride specialization only support stride {1, 1} 
+      if ((ImplicitGemm::kConvolutionalOperator == 
+            cutlass::conv::Operator::kDgrad) && 
+          (ImplicitGemm::ImplicitGemmKernel::Mma::IteratorA::kStrideSupport == 
+            cutlass::conv::StrideSupport::kUnity)) {
+        if (!((conv_problem.stride_h == 1) && (conv_problem.stride_w == 1))) {
+          continue;
+        }
+      }
+
+#if 0 // relax restrictions on analytic strided dgrad
+      // CUTLASS DGRAD's *strided* specialization only support stride >= {2, 2} 
+      if ((ImplicitGemm::kConvolutionalOperator == 
+            cutlass::conv::Operator::kDgrad) && 
+          (ImplicitGemm::ImplicitGemmKernel::Mma::IteratorA::kStrideSupport == 
+            cutlass::conv::StrideSupport::kStrided)) {
+         if (((conv_problem.stride_h == 1) && (conv_problem.stride_w == 1))) {
+           continue;
+         }
+      }
+#endif
+      
+      //
+      // Test
+      //
+      // push back tested problem size to avoid re-running duplicates
+      conv_tested_sizes.push_back(conv_problem);
+
+      // test mode = xcross
+      passed = testbed.run(
+        conv_problem,
+        cutlass::conv::SplitKMode::kSerial);
+    
+      if (!passed) {
+        return false;
+      }
+      
+      // test mode = convolution
+      passed = testbed.run(
+        conv_problem.reset_mode(cutlass::conv::Mode::kConvolution),
+        cutlass::conv::SplitKMode::kSerial);
+    
+      if (!passed) {
+        return false;
+      }
+    }
+  }
+
+  // CUTLASS DGRAD's *strided* specialization does not support split-k mode 
+  if ((ImplicitGemm::kConvolutionalOperator == 
+          cutlass::conv::Operator::kDgrad) && 
+      (ImplicitGemm::ImplicitGemmKernel::Mma::IteratorA::kStrideSupport == 
+        cutlass::conv::StrideSupport::kStrided)) {
+
+    passed = testbed.run(
+      cutlass::conv::Conv2dProblemSize(
+      {1, 56, 56, 8},   // input size (NHWC)
+      {8, 1, 1, 8},     // filter size (KRSC)
+      {0, 0, 0, 0},     // padding (pad_h, _, pad_w, _)
+      {2, 2},           // stride (stride_h, stride_w)
+      {1, 1}),          // dilation (dilation_h, dilation_w)
+      cutlass::conv::SplitKMode::kSerial,
+      cutlass::from_real<typename ImplicitGemm::ElementCompute>(2.0), 
+      cutlass::from_real<typename ImplicitGemm::ElementCompute>(2.0));
+
+    if (!passed) {
+      return false;
+    }
+
+    return passed;
+  }
+
+  // Sweep split-k-slice using serial and prallel reduction with non-unity alpha and non-zero beta for 
+  // a single conv2d problem size. Convolution unit tests take a long time to run so only sweep parameters 
+  // which are abolutely neccessary to catch functional bugs. The below code does provide option to sweep 
+  // alpha and beta for local testing, but only runs one value for alpha and beta.
+  cutlass::conv::Conv2dProblemSize conv2d_split_k_test_size (
+      {1, 17, 11, 288},   // input size (NHWC)
+      {160, 3, 3, 288},   // filter size (KRSC)
+      {1, 1, 1, 1},       // padding (pad_h, _, pad_w, _)
+      {1, 1},             // stride (stride_h, stride_w)
+      {1, 1}              // dilation (dilation_h, dilation_w)
+    );
+
+  cutlass::conv::SplitKMode split_k_modes [] = {
+    cutlass::conv::SplitKMode::kSerial,
+    cutlass::conv::SplitKMode::kParallel,
+  };
+
+  int split_k_slices[] = {
+    1, 2, 3, 4, 201
+  };
+
+  double problem_alpha[] = {
+    2.0
+  };
+
+  double problem_beta[] = {
+    2.0
+  };
+
+  for (auto split_k_mode : split_k_modes) {
+    for (auto split_k_slice : split_k_slices) {
+      for (auto alpha : problem_alpha) {
+        for (auto beta : problem_beta) {
+
+          passed = testbed.run(
+            conv2d_split_k_test_size.reset_split_k_slices(split_k_slice),
+            split_k_mode,
+            cutlass::from_real<typename ImplicitGemm::ElementCompute>(alpha), 
+            cutlass::from_real<typename ImplicitGemm::ElementCompute>(beta));
+
+          if (!passed) {
+            return false;
+          }
+        }
+      }
+    }
+  }
+
+  return passed;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace device
+} // namespace conv
+} // namespace test

test/unit/conv/device/conv2d_with_reduction_testbed.h

@@ -0,0 +1,568 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+/*! \file
+    \brief Implicit GEMM testbed
+*/
+#pragma once
+
+#include "../../common/cutlass_unit_test.h"
+#include "cutlass/cutlass.h"
+
+#include "cutlass/conv/device/implicit_gemm_convolution.h"
+#include "cutlass/reduction/device/reduce_split_k.h"
+#include "cutlass/reduction/thread/reduction_operators.h"
+
+#include "conv2d_problems.h"
+
+#include "cutlass/util/host_tensor.h"
+#include "cutlass/util/reference/host/tensor_fill.h"
+#include "cutlass/util/reference/device/tensor_compare.h"
+#include "cutlass/util/reference/host/tensor_compare.h"
+
+#include "cutlass/util/reference/host/convolution.h"
+#include "cutlass/util/reference/device/convolution.h"
+
+#include "cutlass/core_io.h"
+#include "cutlass/util/tensor_view_io.h"
+
+namespace test {
+namespace conv {
+namespace device {
+
+template <typename Conv2d>
+class TestbedConv2dWithReduction {
+public:
+
+  using ElementA = typename Conv2d::ElementA;
+  using LayoutA = typename Conv2d::LayoutA;
+  using ElementB = typename Conv2d::ElementB;
+  using LayoutB = typename Conv2d::LayoutB;
+  using ElementC = typename Conv2d::ElementC;
+  using LayoutC = typename Conv2d::LayoutC;
+  using ElementAccumulator = typename Conv2d::ElementAccumulator;
+  using ElementCompute = typename Conv2d::ElementCompute;
+  using EpilogueOutputOp = typename Conv2d::EpilogueOutputOp;
+  using ElementT = typename EpilogueOutputOp::ElementTensor;
+
+  static cutlass::conv::Operator const kConvolutionalOperator = Conv2d::kConvolutionalOperator;
+
+public:
+
+  /// Initialization
+  cutlass::Distribution::Kind init_A;
+  cutlass::Distribution::Kind init_B;
+  cutlass::Distribution::Kind init_C;
+  uint64_t seed;
+
+  cutlass::HostTensor<ElementA, LayoutA> tensor_A;
+  cutlass::HostTensor<ElementB, LayoutB> tensor_B;
+  cutlass::HostTensor<ElementC, LayoutC> tensor_C;
+
+  cutlass::HostTensor<ElementAccumulator, cutlass::layout::RowMajor> tensor_Reduction;
+  cutlass::HostTensor<ElementT,           cutlass::layout::RowMajor> tensor_Tensor;
+
+  cutlass::HostTensor<ElementC, LayoutC> tensor_D_computed;
+  cutlass::HostTensor<ElementC, LayoutC> tensor_D_reference;
+
+public:
+
+  TestbedConv2dWithReduction(
+    cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
+    cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
+    cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
+    uint64_t seed_ = 2080
+  ):
+    init_A(init_A_), init_B(init_B_), init_C(init_C_), seed(seed_) {
+
+  }
+
+    /// Helper to initialize a tensor view
+  template <typename Element, typename Layout>
+  void initialize_tensor(
+    cutlass::TensorView<Element, Layout> view, 
+    cutlass::Distribution::Kind dist_kind,
+    uint64_t seed) {
+
+    if (dist_kind == cutlass::Distribution::Uniform) {
+
+      int scope;
+      int bits = cutlass::sizeof_bits<Element>::value;
+
+      if (bits <= 8) {
+        scope = 2;
+      }
+      else if (bits == 16) {
+        scope = 3;
+      }
+      else {
+        scope = 8;
+      }
+      cutlass::reference::host::TensorFillRandomUniform(
+        view, seed, scope, -scope, 0);
+    } 
+    else if (dist_kind == cutlass::Distribution::Identity) {
+
+      cutlass::reference::host::TensorFillIdentity(view);
+    } 
+    else if (dist_kind == cutlass::Distribution::Gaussian) {
+
+      cutlass::reference::host::TensorFillRandomGaussian(view, seed, 0, 0.5);
+    }
+    else if (dist_kind == cutlass::Distribution::Sequential) {
+
+      cutlass::reference::host::BlockFillSequential(view.data(), view.capacity());
+    } 
+    else {
+    }
+  }
+
+  void initialize(
+    cutlass::conv::Conv2dProblemSize const &problem_size, uint64_t seed = 2019) {
+        
+    tensor_A.resize(implicit_gemm_tensor_a_extent(kConvolutionalOperator, problem_size));
+    tensor_B.resize(implicit_gemm_tensor_b_extent(kConvolutionalOperator, problem_size));
+    tensor_C.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
+
+    tensor_Reduction.resize({
+      (problem_size.N * problem_size.P * problem_size.Q), 
+      (problem_size.K - 1 + Conv2d::ThreadblockShape::kN) / Conv2d::ThreadblockShape::kN
+    });
+
+    tensor_Tensor.resize({(problem_size.N * problem_size.P * problem_size.Q), problem_size.K});
+
+    tensor_D_computed.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
+    tensor_D_reference.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
+
+    initialize_tensor(tensor_A.host_view(), init_A, seed); 
+    initialize_tensor(tensor_B.host_view(), init_B, seed * 17); 
+    initialize_tensor(tensor_C.host_view(), init_C, seed * 39);
+    
+    tensor_A.sync_device();
+    tensor_B.sync_device();
+    tensor_C.sync_device();
+    tensor_D_computed.sync_device();
+    tensor_D_reference.sync_device();
+  }
+
+  bool sufficient() const {
+    //
+    // Determine SMEM requirements and waive if not satisfied
+    //
+
+    int smem_size = int(sizeof(typename Conv2d::ImplicitGemmKernel::SharedStorage));
+
+    cudaDeviceProp properties;
+    int device_idx;
+    cudaError_t result = cudaGetDevice(&device_idx);
+
+    if (result != cudaSuccess) {
+      throw std::runtime_error("cudaGetDevice() API call failed.");
+    }
+
+    result = cudaGetDeviceProperties(&properties, device_idx);
+
+    if (result != cudaSuccess) {
+      throw std::runtime_error("cudaGetDeviceProperties() failed");
+    }
+
+    if (properties.sharedMemPerMultiprocessor < smem_size) {
+      return false;
+    }
+
+    return true;
+  }
+
+  /// Executes one test
+  bool run(
+    cutlass::conv::Conv2dProblemSize const &problem_size,
+    cutlass::conv::SplitKMode const &split_k_mode = cutlass::conv::SplitKMode::kSerial,
+    ElementCompute alpha = ElementCompute(1),
+    ElementCompute beta = ElementCompute(0)) {
+
+    // Waive test if insufficient CUDA device
+    if (!sufficient()) {
+      if (CUTLASS_TEST_UNIT_ENABLE_WARNINGS) {
+        std::cerr << "Test waived due to insufficient CUDA device." << std::endl;
+      }
+      return true;
+    }
+
+#if 0 //display conv2d problem size for debugging
+    std::cout << problem_size << std::endl
+              << "alpha, beta: (" << alpha << ", " << beta << ")" << std::endl
+              << "split_k_mode: " << ((split_k_mode == cutlass::conv::SplitKMode::kSerial) ? "(serial)" : "(parallel)") << std::endl
+              << std::endl;
+#endif
+
+    initialize(problem_size);
+
+    // configure the operator
+    Conv2d conv2d_op;
+
+    typename Conv2d::Arguments conv2d_args(
+      problem_size,
+      tensor_A.device_ref(),
+      tensor_B.device_ref(),
+      tensor_C.device_ref(),
+      tensor_D_computed.device_ref(),
+      {alpha, beta},
+      split_k_mode,
+      tensor_Reduction.device_data(),
+      tensor_Tensor.device_data(),
+      static_cast<int>(tensor_Reduction.stride()[0]),
+      static_cast<int>(tensor_Tensor.stride()[0])
+    );
+
+    // find workspace requirement for parallel split-k reduction
+    size_t workspace_size = Conv2d::get_workspace_size(conv2d_args);
+
+    cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
+
+    cutlass::Status status = conv2d_op.initialize(conv2d_args, workspace.get());
+
+    if (status != cutlass::Status::kSuccess) {
+      cudaError_t error = cudaGetLastError();
+      std::cerr << "This test is not supported: " << cudaGetErrorString(error) << "\n";
+      return true;
+    }
+
+    // conv2d operation with parallel split-k-mode
+    if (split_k_mode == cutlass::conv::SplitKMode::kParallel) {
+
+      // conv2d output is written to workspace in global memory
+      conv2d_args.ref_D.reset(reinterpret_cast<ElementC*>(workspace.get()));
+      // accumulate mma for each cta in k-dimension (1.0 * A * B)
+      conv2d_args.output_op = {ElementCompute(1), ElementCompute(0)}; 
+      // update conv2d operator arguments
+      status = conv2d_op.update(conv2d_args, workspace.get());
+    }
+    
+    EXPECT_TRUE(status == cutlass::Status::kSuccess);
+    if (status != cutlass::Status::kSuccess) {
+      return false;
+    }
+
+    // run conv2d operator
+    status = conv2d_op();
+    
+    EXPECT_TRUE(status == cutlass::Status::kSuccess);
+    if (status != cutlass::Status::kSuccess) {
+      return false;
+    }
+
+    bool passed = false;
+
+    cudaError_t result = cudaDeviceSynchronize();
+    EXPECT_EQ(result, cudaSuccess) << " device reference error: " 
+                                   << cudaGetErrorString(result);
+
+    tensor_D_computed.sync_host();
+
+#if CUTLASS_CONV_TEST_UNIT_REFERENCE_DEVICE_ENABLED
+
+    cutlass::reference::device::Conv2d<
+      ElementA,
+      LayoutA,
+      ElementB,
+      LayoutB,
+      ElementC,
+      LayoutC,
+      ElementCompute,
+      ElementAccumulator 
+    >(
+      kConvolutionalOperator,
+      problem_size,
+      tensor_A.device_ref(),
+      tensor_B.device_ref(),
+      tensor_C.device_ref(),
+      tensor_D_reference.device_ref(),
+      alpha, 
+      beta);
+
+    // sync host (copy device data to host) for dumping error output in case of mismatches
+    tensor_D_reference.sync_host();
+    
+#else 
+
+    cutlass::reference::host::Conv2d<
+      ElementA,
+      LayoutA,
+      ElementB,
+      LayoutB,
+      ElementC,
+      LayoutC,
+      ElementCompute,
+      ElementAccumulator
+    >(
+      kConvolutionalOperator,
+      problem_size,
+      tensor_A.host_ref(),
+      tensor_B.host_ref(),
+      tensor_C.host_ref(),
+      tensor_D_reference.host_ref(),
+      alpha, 
+      beta);
+
+#endif
+    passed = cutlass::reference::host::TensorEquals(
+      tensor_D_computed.host_view(), 
+      tensor_D_reference.host_view());
+
+    EXPECT_TRUE(passed);
+
+    if (!passed) {
+      std::stringstream fname;
+
+      fname << "error_Conv2d_ImplicitGemm_device_"
+        << (split_k_mode == cutlass::conv::SplitKMode::kSerial ? "serial_reduction_" : "parallel_reduction_")
+        << (Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kFprop ? "fprop_" :
+            (Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ? "dgrad_" : "wgrad_")) 
+        << "nhwc_"
+        << problem_size.N << "x"
+        << problem_size.H << "x"
+        << problem_size.W << "x"
+        << problem_size.C 
+        << "_krsc_"
+        << problem_size.K << "x"
+        << problem_size.R << "x"
+        << problem_size.S << "x"
+        << problem_size.C 
+        << "_padding_" 
+        << problem_size.pad_h << "x"
+        << problem_size.pad_w 
+        << "_stride_"  
+        << problem_size.stride_h << "x"
+        << problem_size.stride_w 
+        << "_dilation_"
+        << problem_size.dilation_h << "x"
+        << problem_size.dilation_w << "_"
+        << (problem_size.mode == cutlass::conv::Mode::kCrossCorrelation ? "xcorr_" : "conv_")
+        << Conv2d::ThreadblockShape::kM << "x"  
+        << Conv2d::ThreadblockShape::kN << "x"  
+        << Conv2d::ThreadblockShape::kK << "_"
+        << Conv2d::WarpShape::kM << "x"  
+        << Conv2d::WarpShape::kN << "x"  
+        << Conv2d::WarpShape::kK << ".txt";
+
+      std::cout << fname.str() << std::endl;
+
+      std::ofstream results(fname.str());
+
+      results << problem_size << std::endl;
+
+      results
+        << "\nA:\n" << tensor_A.host_view() << "\n"
+        << "\nB:\n" << tensor_B.host_view() << "\n"
+        << "\nC:\n" << tensor_C.host_view() << "\n"
+        << "\nD reference:\n" << tensor_D_reference.host_view() << "\n"
+        << "\nD computed:\n" << tensor_D_computed.host_view() << "\n";
+
+    }
+
+    return passed;
+  }
+
+};
+
+/////////////////////////////////////////////////////////////////////////////////////////////////////////
+// TestAllConv: Runs cutlass::conv::device::ImplicitGemmConvolution operator and compares it with reference
+// TestAllConv runs conv operator on default conv problem sizes from test::conv::device::TestbedConv2dProblemSizes
+// Additionaly, each conv2d test can provide conv problem sizes (conv_test_sizes) and blacklist of sizes 
+// (conv_blacklist_sizes)
+/////////////////////////////////////////////////////////////////////////////////////////////////////////////
+template <typename ImplicitGemm>
+bool TestAllConv2dWithReduction(
+  const Conv2dProblemVector & conv_test_sizes = Conv2dProblemVector(),
+  const Conv2dProblemVector & conv_blacklist_sizes = Conv2dProblemVector()) {
+
+  bool passed = true;
+
+  //
+  // Testbed object
+  //
+
+  TestbedConv2dWithReduction<ImplicitGemm> testbed;
+
+  //
+  // Get conv problem sizes to run conv operator 
+  //
+  TestbedConv2dProblemSizes conv_problems(128/cutlass::sizeof_bits<typename ImplicitGemm::ElementA>::value);
+
+  // Vector of conv2d problem sizes to avoid duplicate runs
+  Conv2dProblemVector conv_tested_sizes;
+
+  Conv2dProblemVector const *problem_vectors[] = {
+    &conv_test_sizes,                               // run user specified sizes
+    &conv_problems.conv2d_default_sizes,            // run default and cudnn bug sizes
+    &conv_problems.conv2d_resnet50_sizes,           // run resnet50 sizes
+#if CUTLASS_CONV_UNIT_TEST_RIGOROUS_SIZE_ENABLED 
+    &conv_problems.conv2d_rigorous_sizes,           // run large and rigorous sizes if enabled
+#endif
+  };
+
+  // Sweep conv2d problem sizes (split-k-mode=kSerial, split-k-slice=1, alpha=1.0, beta=0.0)
+  for (Conv2dProblemVector const * problem_vector : problem_vectors) {
+
+    //  Run conv testbed on default convolution sizes
+    for(auto conv_problem : *problem_vector) {
+
+      // Skip blacklist and avoid duplicate problem sizes
+      if (std::find(conv_blacklist_sizes.begin(), conv_blacklist_sizes.end(), conv_problem) != conv_blacklist_sizes.end() ||
+          std::find(conv_tested_sizes.begin(), conv_tested_sizes.end(), conv_problem) != conv_tested_sizes.end()) {
+        continue;
+      }
+
+      //
+      // Procedurally disable certain cases
+      //
+  
+      // CUTLASS DGRAD's *unity* stride specialization only support stride {1, 1} 
+      if ((ImplicitGemm::kConvolutionalOperator == 
+            cutlass::conv::Operator::kDgrad) && 
+          (ImplicitGemm::ImplicitGemmKernel::Mma::IteratorA::kStrideSupport == 
+            cutlass::conv::StrideSupport::kUnity)) {
+        if (!((conv_problem.stride_h == 1) && (conv_problem.stride_w == 1))) {
+          continue;
+        }
+      }
+
+#if 0 // relax restrictions on analytic strided dgrad
+      // CUTLASS DGRAD's *strided* specialization only support stride >= {2, 2} 
+      if ((ImplicitGemm::kConvolutionalOperator == 
+            cutlass::conv::Operator::kDgrad) && 
+          (ImplicitGemm::ImplicitGemmKernel::Mma::IteratorA::kStrideSupport == 
+            cutlass::conv::StrideSupport::kStrided)) {
+         if (((conv_problem.stride_h == 1) && (conv_problem.stride_w == 1))) {
+           continue;
+         }
+      }
+#endif
+      
+      //
+      // Test
+      //
+      // push back tested problem size to avoid re-running duplicates
+      conv_tested_sizes.push_back(conv_problem);
+
+      // test mode = xcross
+      passed = testbed.run(
+        conv_problem,
+        cutlass::conv::SplitKMode::kSerial);
+    
+      if (!passed) {
+        return false;
+      }
+      
+      // test mode = convolution
+      passed = testbed.run(
+        conv_problem.reset_mode(cutlass::conv::Mode::kConvolution),
+        cutlass::conv::SplitKMode::kSerial);
+    
+      if (!passed) {
+        return false;
+      }
+    }
+  }
+
+  // CUTLASS DGRAD's *strided* specialization does not support split-k mode 
+  if ((ImplicitGemm::kConvolutionalOperator == 
+          cutlass::conv::Operator::kDgrad) && 
+      (ImplicitGemm::ImplicitGemmKernel::Mma::IteratorA::kStrideSupport == 
+        cutlass::conv::StrideSupport::kStrided)) {
+
+    passed = testbed.run(
+      cutlass::conv::Conv2dProblemSize(
+      {1, 56, 56, 8},   // input size (NHWC)
+      {8, 1, 1, 8},     // filter size (KRSC)
+      {0, 0, 0, 0},     // padding (pad_h, _, pad_w, _)
+      {2, 2},           // stride (stride_h, stride_w)
+      {1, 1}),          // dilation (dilation_h, dilation_w)
+      cutlass::conv::SplitKMode::kSerial,
+      cutlass::from_real<typename ImplicitGemm::ElementCompute>(2.0), 
+      cutlass::from_real<typename ImplicitGemm::ElementCompute>(2.0));
+
+    if (!passed) {
+      return false;
+    }
+
+    return passed;
+  }
+
+  // Sweep split-k-slice using serial and prallel reduction with non-unity alpha and non-zero beta for 
+  // a single conv2d problem size. Convolution unit tests take a long time to run so only sweep parameters 
+  // which are abolutely neccessary to catch functional bugs. The below code does provide option to sweep 
+  // alpha and beta for local testing, but only runs one value for alpha and beta.
+  cutlass::conv::Conv2dProblemSize conv2d_split_k_test_size (
+      {1, 17, 11, 288},   // input size (NHWC)
+      {160, 3, 3, 288},   // filter size (KRSC)
+      {1, 1, 1, 1},       // padding (pad_h, _, pad_w, _)
+      {1, 1},             // stride (stride_h, stride_w)
+      {1, 1}              // dilation (dilation_h, dilation_w)
+    );
+
+  cutlass::conv::SplitKMode split_k_modes [] = {
+    cutlass::conv::SplitKMode::kSerial,
+    cutlass::conv::SplitKMode::kParallel,
+  };
+
+  int split_k_slices[] = {
+    1, 2, 3, 4, 201
+  };
+
+  double problem_alpha[] = {
+    2.0
+  };
+
+  double problem_beta[] = {
+    2.0
+  };
+
+  for (auto split_k_mode : split_k_modes) {
+    for (auto split_k_slice : split_k_slices) {
+      for (auto alpha : problem_alpha) {
+        for (auto beta : problem_beta) {
+
+          passed = testbed.run(
+            conv2d_split_k_test_size.reset_split_k_slices(split_k_slice),
+            split_k_mode,
+            cutlass::from_real<typename ImplicitGemm::ElementCompute>(alpha), 
+            cutlass::from_real<typename ImplicitGemm::ElementCompute>(beta));
+
+          if (!passed) {
+            return false;
+          }
+        }
+      }
+    }
+  }
+
+  return passed;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace device
+} // namespace conv
+} // namespace test

test/unit/conv/device/conv3d_testbed.h

@@ -81,7 +81,8 @@ public:
   >;
 
   using ReductionDevice = cutlass::reduction::device::ReduceSplitK<ReductionKernel>;
-
+  using ReductionStrideIndex = typename ReductionDevice::StrideIndex;
+  
 public:
 
   /// Initialization
@@ -281,10 +282,20 @@ public:
         cutlass::conv::implicit_gemm_problem_size(kConvolutionalOperator, problem_size).mn(),
         problem_size.split_k_slices,
         cutlass::conv::implicit_gemm_tensor_c_size(kConvolutionalOperator, problem_size),
-        {reinterpret_cast<ElementAccumulator*> (workspace.get()), tensor_C.stride(Conv3d::ImplicitGemmKernel::kTensorCStrideIdx)},
-        {tensor_D_computed.device_data(), tensor_C.stride(Conv3d::ImplicitGemmKernel::kTensorCStrideIdx)},
-        {tensor_C.device_data(), tensor_C.stride(Conv3d::ImplicitGemmKernel::kTensorCStrideIdx)},
-        {alpha, beta} // apply alpha, beta to obtain the following equation alpha * ReduceAdd(A * B) + beta * C 
+        {
+          reinterpret_cast<ElementAccumulator*> (workspace.get()),
+          ReductionStrideIndex(tensor_C.stride()[Conv3d::ImplicitGemmKernel::kTensorCStrideIdx])
+        },
+        {
+          tensor_D_computed.device_data(),
+          ReductionStrideIndex(tensor_C.stride()[Conv3d::ImplicitGemmKernel::kTensorCStrideIdx])
+        },
+        {
+          tensor_C.device_data(),
+          ReductionStrideIndex(tensor_C.stride()[Conv3d::ImplicitGemmKernel::kTensorCStrideIdx])
+        },
+        // apply alpha, beta to obtain the following equation alpha * ReduceAdd(A * B) + beta * C 
+        {alpha, beta}
       );
 
       status = reduction_op.initialize(reduction_args, nullptr);
@@ -304,6 +315,38 @@ public:
     }
     bool passed = false;
 
+    cudaError_t result = cudaDeviceSynchronize();
+    EXPECT_EQ(result, cudaSuccess) << " device reference error: " 
+                                   << cudaGetErrorString(result);
+
+    tensor_D_computed.sync_host();
+
+#if CUTLASS_CONV_TEST_UNIT_REFERENCE_DEVICE_ENABLED
+    
+    cutlass::reference::device::Conv3d<
+      ElementA,
+      LayoutA,
+      ElementB,
+      LayoutB,
+      ElementC,
+      LayoutC,
+      ElementAccumulator,
+      ElementCompute
+    >(
+      kConvolutionalOperator,
+      problem_size,
+      tensor_A.device_ref(),
+      tensor_B.device_ref(),
+      tensor_C.device_ref(),
+      tensor_D_reference.device_ref(),
+      alpha, 
+      beta
+    );
+
+    // sync host (copy device data to host) for dumping error output in case of mismatches
+    tensor_D_reference.sync_host();
+    
+#else
     cutlass::reference::host::Conv3d<
       ElementA,
       LayoutA,
@@ -323,8 +366,7 @@ public:
       alpha,
       beta
     );
-
-    tensor_D_computed.sync_host();
+#endif
 
     passed = cutlass::reference::host::TensorEquals(
       tensor_D_computed.host_view(), 

test/unit/core/complex.cu

@@ -32,6 +32,7 @@
 #include "../common/cutlass_unit_test.h"
 
 #include "cutlass/complex.h"
+#include "cutlass/constants.h"
 #include "cutlass/numeric_conversion.h"
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
@@ -85,6 +86,42 @@ TEST(complex, f16_to_f32_conversion) {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
+TEST(complex, exp_f32) {
+  
+  cutlass::complex<float> Z[] = {
+    {1, 1},
+    {2   ,  cutlass::constants::pi<float>()/2.0f   },
+    {0.5f,  cutlass::constants::pi<float>()        },
+    {0.25f,  cutlass::constants::pi<float>()*3/4.0f },
+    {0, 0},
+  };
+
+  cutlass::complex<double> Expected[] = {
+    {1.4686939399158851, 2.2873552871788423}, 
+    {4.524491950137825e-16, 7.38905609893065},
+    {-1.6487212707001282, 2.019101226849069e-16}, 
+    {-0.9079430793557842, 0.9079430793557843},
+    {1, 0}
+  };
+
+  double tolerance = 0.00001;
+
+  for (int i = 0; cutlass::real(Z[i]); ++i) {
+    double e_r = cutlass::real(Expected[i]);
+    double e_i = cutlass::real(Expected[i]);
+
+    cutlass::complex<float> got = cutlass::exp(Z[i]);
+    float g_r = cutlass::real(got);
+    float g_i = cutlass::real(got);
+
+    EXPECT_TRUE(
+      std::abs(g_r - e_r) < tolerance && std::abs(g_i - e_i) < tolerance
+    ) << "Expected(" << Expected[i] << "), Got(" << got << ")";
+  }
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
 namespace test {
 
   /// Thorough testing for basic complex math operators. Uses std::complex as a reference.

test/unit/core/functional.cu

@@ -29,6 +29,7 @@
 #include "../common/cutlass_unit_test.h"
 
 #include "cutlass/functional.h"
+#include "cutlass/core_io.h"
 
 #include "cutlass/layout/matrix.h"
 #include "cutlass/util/host_tensor.h"
@@ -78,16 +79,16 @@ __global__ void trinary_operator(
 
   Operator op;
 
-  Element a_x = *a;
-  Element b_x = *b;
-  Element c_x = *c;
+  Element a_x = a[blockIdx.x];
+  Element b_x = b[blockIdx.x];
+  Element c_x = c[blockIdx.x];
 
   CUTLASS_PRAGMA_NO_UNROLL
   for (int i = 0; i < Iterations; ++i) {
     c_x = op(a_x, b_x, c_x);
   }
   
-  *d = c_x;
+  d[blockIdx.x] = c_x;
 }
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
@@ -421,3 +422,67 @@ TEST(Functional, multiply_add_bf16x17) {
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
 
+template <typename T>
+cutlass::Quaternion<T> random_quaternion(int range) {
+  return cutlass::Quaternion<T>{
+    T((rand() % range * 2) - range),
+    T((rand() % range * 2) - range),
+    T((rand() % range * 2) - range),
+    T((rand() % range * 2) - range)
+  };
+}
+
+template <typename T>
+void Functional_multiply_add_QuaternionT() {
+
+  using Element = cutlass::Quaternion<T>;
+  using Operator = cutlass::multiply_add<Element, Element, Element>;
+  using HostTensor = cutlass::HostTensor<Element, cutlass::layout::RowMajor>;
+
+  int const kM = 128;
+  int const kRange = 8;
+
+  HostTensor A({kM, 1});
+  HostTensor B({kM, 1});
+  HostTensor C({kM, 1});
+  HostTensor D({kM, 1});
+
+  srand(2021);
+
+  for (int m = 0; m < kM; ++m) {
+    A.at({m, 0}) = random_quaternion<T>(kRange);
+    B.at({m, 0}) = random_quaternion<T>(kRange);
+    C.at({m, 0}) = random_quaternion<T>(kRange);
+  }
+
+  A.sync_device();
+  B.sync_device();
+  C.sync_device();
+  D.sync_device();
+
+  test::core::kernel::trinary_operator<Element, Operator><<< dim3(kM,1), dim3(1,1) >>>(
+    D.device_data(),
+    A.device_data(),
+    B.device_data(),
+    C.device_data()
+  );
+
+  D.sync_host();
+  
+  for (int m = 0; m < kM; ++m) {
+
+    Element a = A.at({m, 0});
+    Element b = B.at({m, 0});
+    Element c = C.at({m, 0});
+    Element got = D.at({m, 0});
+    Element expected = a * b + c;
+
+    EXPECT_TRUE(got == expected);
+  }
+}
+
+TEST(Functional, multiply_add_quaternion_f32) {
+  Functional_multiply_add_QuaternionT<float>();
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////

test/unit/core/matrix.cu

@@ -32,6 +32,7 @@
 #include "../common/cutlass_unit_test.h"
 
 #include "cutlass/matrix.h"
+#include "cutlass/core_io.h"
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
 

test/unit/epilogue/thread/linear_combination.cu

@@ -122,13 +122,14 @@ TEST(Epilogue_thread_linear_combination, device_side_f16_f32_ptr) {
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
 
+
 TEST(Epilogue_thread_linear_combination_gelu, device_side_f16_f16_ptr) {
 
   using Element = cutlass::half_t;
   using ElementOutput = cutlass::half_t;
   int const kCount = 8;
 
-  using LinearCombination = cutlass::epilogue::thread::LinearCombinationGELU<
+  using LinearCombinationGELU = cutlass::epilogue::thread::LinearCombinationGELU<
     ElementOutput,
     kCount,
     Element,
@@ -137,9 +138,9 @@ TEST(Epilogue_thread_linear_combination_gelu, device_side_f16_f16_ptr) {
   Element alpha = Element(1);
   Element beta = Element(0);
 
-  typename LinearCombination::Params params(&alpha, &beta);
+  typename LinearCombinationGELU::Params params(&alpha, &beta);
 
-  LinearCombination linear_combination_op(params);
+  LinearCombinationGELU linear_combination_op(params);
 
   cutlass::Array<Element, kCount> accum;
 
@@ -157,4 +158,4 @@ TEST(Epilogue_thread_linear_combination_gelu, device_side_f16_f16_ptr) {
   }
 }
 
-/////////////////////////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////////////////////////

test/unit/epilogue/threadblock/CMakeLists.txt

@@ -32,4 +32,5 @@ cutlass_test_unit_add_executable(
   epilogue_volta_tensor_op.cu
   epilogue_wmma_tensor_op_sm70.cu
   epilogue_planar_complex.cu
+  epilogue_with_reduction_tensor_op.cu
 )

test/unit/epilogue/threadblock/epilogue_simt.cu

@@ -32,6 +32,7 @@
 
 #include "cutlass/aligned_buffer.h"
 #include "cutlass/complex.h"
+#include "cutlass/quaternion.h"
 
 #include "cutlass/gemm/warp/mma_simt.h"
 #include "cutlass/gemm/warp/mma_simt_policy.h"
@@ -1088,4 +1089,80 @@ TEST(SM50_Epilogue_threadblock_epilogue, simt_complex_f64_128x128_32x64x8) {
   EXPECT_TRUE(passed);
 }
 
-///////////////////////////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////////////////////////
+//
+// Quaternion-valued single-precision
+//
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM50_Epilogue_threadblock_epilogue, simt_quaternion_f32_32x64_32x64x8) {
+
+  //
+  // Define the warp-level matrix multiply
+  //
+
+  using Element = cutlass::Quaternion<float>;
+  using ElementOutput = Element;
+  using ElementAccumulator = Element;
+  using ElementCompute = Element;
+  int const kElementsPerAccess = 1;
+
+  using Shape = cutlass::gemm::GemmShape<32, 64, 8>;
+  using WarpShape = cutlass::gemm::GemmShape<32, 64, 8>;
+  
+  using ElementC = ElementAccumulator;
+  using LayoutA = cutlass::layout::ColumnMajor;
+  using LayoutB = cutlass::layout::RowMajor;
+  using LayoutC = cutlass::layout::RowMajor;
+
+  using ElementOutput = Element;
+  using ElementAccumulator = Element;
+  using ElementCompute = Element;
+
+  using WarpMmaSimt = cutlass::gemm::warp::MmaSimt<
+    WarpShape,
+    Element,
+    LayoutA,
+    Element,
+    LayoutB,
+    Element,
+    LayoutC,
+    cutlass::gemm::warp::MmaSimtPolicy<
+      cutlass::MatrixShape<4, 8>,
+      cutlass::layout::RowMajorInterleaved<2>,
+      cutlass::gemm::GemmShape<2, 2, 1>
+    >
+  >;
+
+  //
+  // Output operator
+  //
+
+  using OutputOp = cutlass::epilogue::thread::LinearCombination<
+    ElementOutput,
+    kElementsPerAccess,
+    ElementAccumulator,
+    ElementCompute
+  >;
+
+  //
+  // Define the epilogue
+  //
+
+  using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueSimt<
+    Shape,
+    WarpMmaSimt,
+    OutputOp,
+    kElementsPerAccess
+  >::Epilogue;
+
+  //
+  // Instantiate epilogue
+  //
+
+  EpilogueTestbed<Epilogue> testbed;
+
+  bool passed = testbed.run_all();
+
+  EXPECT_TRUE(passed);
+}

test/unit/epilogue/threadblock/epilogue_with_reduction_tensor_op.cu

@@ -0,0 +1,875 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+/*! \file
+
+  \brief Unit tests for thread-level GEMM
+*/
+
+#include <fstream>
+
+#include "../../common/cutlass_unit_test.h"
+
+#include "cutlass/aligned_buffer.h"
+#include "cutlass/half.h"
+
+#include "cutlass/epilogue/thread/linear_combination_drelu.h"
+#include "cutlass/gemm/warp/default_mma_tensor_op.h"
+#include "cutlass/epilogue/threadblock/default_epilogue_with_reduction.h"
+#include "cutlass/epilogue/threadblock/epilogue_with_reduction.h"
+
+#include "cutlass/util/host_tensor.h"
+#include "cutlass/util/tensor_view_io.h"
+#include "cutlass/util/reference/host/tensor_fill.h"
+
+#include "epilogue_with_reduction_testbed.h"
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+//
+// Disable selected tests on CUDA 11.1
+//
+//
+#define ENABLE_BLOCKED_TESTS (!(__CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ == 1))
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM75_Epilogue_with_reduction_threadblock, f16_tensor_op_64x64_64x64x8) {
+
+  //
+  // Define the warp-level matrix multiply
+  //
+
+  using ElementOutput = cutlass::half_t;
+  using ElementAccumulator = float;
+  using ElementCompute = float;
+  int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
+  int const kPartitionsK = 1;
+  
+  using Shape = cutlass::gemm::GemmShape<64, 64, 8>;
+  using WarpShape = cutlass::gemm::GemmShape<64, 64, 8>;
+  using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
+  using Element = cutlass::half_t;
+  using ElementC = ElementAccumulator;
+  using LayoutA = cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutB = cutlass::layout::RowMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutC = cutlass::layout::RowMajor;
+
+  using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
+      WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
+      LayoutC>::Type;
+
+  //
+  // Output operator
+  //
+
+  using OutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
+    ElementAccumulator,
+    ElementAccumulator,
+    ElementOutput,
+    ElementOutput,
+    kElementsPerAccess
+  >;
+
+  using ReductionOp = cutlass::plus<ElementAccumulator>;
+
+  //
+  // Define the epilogue
+  //
+
+  using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueWithReductionTensorOp<
+    Shape,
+    WarpMmaTensorOp,
+    kPartitionsK,
+    ElementOutput,
+    OutputOp,
+    ReductionOp,
+    kElementsPerAccess
+  >::Epilogue;
+
+  //
+  // Instantiate epilogue
+  //
+
+  EpilogueWithReductionTestbed<Epilogue> testbed;
+
+  bool passed = testbed.run_all();
+
+  EXPECT_TRUE(passed);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM75_Epilogue_with_reduction_threadblock, f32_tensor_op_64x64_64x64x8) {
+
+  //
+  // Define the warp-level matrix multiply
+  //
+
+  using ElementOutput = float;
+  using ElementAccumulator = float;
+  using ElementCompute = float;
+  int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
+  int const kPartitionsK = 1;
+  
+  using Shape = cutlass::gemm::GemmShape<64, 64, 8>;
+  using WarpShape = cutlass::gemm::GemmShape<64, 64, 8>;
+  using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
+  using Element = cutlass::half_t;
+  using ElementC = ElementAccumulator;
+  using LayoutA = cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutB = cutlass::layout::RowMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutC = cutlass::layout::RowMajor;
+
+  using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
+      WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
+      LayoutC>::Type;
+
+  //
+  // Output operator
+  //
+
+  using OutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
+    ElementAccumulator,
+    ElementAccumulator,
+    ElementOutput,
+    ElementOutput,
+    kElementsPerAccess
+  >;
+
+  using ReductionOp = cutlass::plus<ElementAccumulator>;
+
+  //
+  // Define the epilogue
+  //
+
+  using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueWithReductionTensorOp<
+    Shape,
+    WarpMmaTensorOp,
+    kPartitionsK,
+    ElementOutput,
+    OutputOp,
+    ReductionOp,
+    kElementsPerAccess
+  >::Epilogue;
+
+  //
+  // Instantiate epilogue
+  //
+
+  EpilogueWithReductionTestbed<Epilogue> testbed;
+
+  bool passed = testbed.run_all();
+
+  EXPECT_TRUE(passed);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM75_Epilogue_with_reduction_threadblock, f32_tensor_op_128x128_64x64x8) {
+
+  //
+  // Define the warp-level matrix multiply
+  //
+
+  using ElementOutput = float;
+  using ElementAccumulator = float;
+  using ElementCompute = float;
+  int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
+  int const kPartitionsK = 1;
+  
+  using Shape = cutlass::gemm::GemmShape<128, 128, 8>;
+  using WarpShape = cutlass::gemm::GemmShape<64, 64, 8>;
+  using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
+  using Element = cutlass::half_t;
+  using ElementC = ElementAccumulator;
+  using LayoutA = cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutB = cutlass::layout::RowMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutC = cutlass::layout::RowMajor;
+
+  using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
+      WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
+      LayoutC>::Type;
+
+  //
+  // Output operator
+  //
+
+  using OutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
+    ElementAccumulator,
+    ElementAccumulator,
+    ElementOutput,
+    ElementOutput,
+    kElementsPerAccess
+  >;
+
+  using ReductionOp = cutlass::plus<ElementAccumulator>;
+
+  //
+  // Define the epilogue
+  //
+
+  using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueWithReductionTensorOp<
+    Shape,
+    WarpMmaTensorOp,
+    kPartitionsK,
+    ElementOutput,
+    OutputOp,
+    ReductionOp,
+    kElementsPerAccess
+  >::Epilogue;
+
+  //
+  // Instantiate epilogue
+  //
+
+  EpilogueWithReductionTestbed<Epilogue> testbed;
+
+  bool passed = testbed.run_all();
+
+  EXPECT_TRUE(passed);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM75_Epilogue_with_reduction_threadblock, f16_tensor_op_128x128_64x64x8) {
+
+  //
+  // Define the warp-level matrix multiply
+  //
+
+  using ElementOutput = cutlass::half_t;
+  using ElementAccumulator = float;
+  using ElementCompute = float;
+  int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
+  int const kPartitionsK = 1;
+  
+  using Shape = cutlass::gemm::GemmShape<128, 128, 8>;
+  using WarpShape = cutlass::gemm::GemmShape<64, 64, 8>;
+  using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
+  using Element = cutlass::half_t;
+  using ElementC = ElementAccumulator;
+  using LayoutA = cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutB = cutlass::layout::RowMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutC = cutlass::layout::RowMajor;
+
+  using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
+      WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
+      LayoutC>::Type;
+
+  //
+  // Output operator
+  //
+
+  using OutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
+    ElementAccumulator,
+    ElementAccumulator,
+    ElementOutput,
+    ElementOutput,
+    kElementsPerAccess
+  >;
+
+  using ReductionOp = cutlass::plus<ElementAccumulator>;
+
+  //
+  // Define the epilogue
+  //
+
+  using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueWithReductionTensorOp<
+    Shape,
+    WarpMmaTensorOp,
+    kPartitionsK,
+    ElementOutput,
+    OutputOp,
+    ReductionOp,
+    kElementsPerAccess
+  >::Epilogue;
+
+  //
+  // Instantiate epilogue
+  //
+
+  EpilogueWithReductionTestbed<Epilogue> testbed;
+
+  bool passed = testbed.run_all();
+
+  EXPECT_TRUE(passed);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM75_Epilogue_with_reduction_threadblock, f32_tensor_op_128x64_64x32x8) {
+
+  //
+  // Define the warp-level matrix multiply
+  //
+
+  using ElementOutput = float;
+  using ElementAccumulator = float;
+  using ElementCompute = float;
+  int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
+  int const kPartitionsK = 1;
+  
+  using Shape = cutlass::gemm::GemmShape<128, 64, 8>;
+  using WarpShape = cutlass::gemm::GemmShape<64, 32, 8>;
+  using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
+  using Element = cutlass::half_t;
+  using ElementC = ElementAccumulator;
+  using LayoutA = cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutB = cutlass::layout::RowMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutC = cutlass::layout::RowMajor;
+
+  using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
+      WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
+      LayoutC>::Type;
+
+  //
+  // Output operator
+  //
+
+  using OutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
+    ElementAccumulator,
+    ElementAccumulator,
+    ElementOutput,
+    ElementOutput,
+    kElementsPerAccess
+  >;
+
+  using ReductionOp = cutlass::plus<ElementAccumulator>;
+
+  //
+  // Define the epilogue
+  //
+
+  using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueWithReductionTensorOp<
+    Shape,
+    WarpMmaTensorOp,
+    kPartitionsK,
+    ElementOutput,
+    OutputOp,
+    ReductionOp,
+    kElementsPerAccess
+  >::Epilogue;
+
+  //
+  // Instantiate epilogue
+  //
+
+  EpilogueWithReductionTestbed<Epilogue> testbed;
+
+  bool passed = testbed.run_all();
+
+  EXPECT_TRUE(passed);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+#if ENABLE_BLOCKED_TESTS
+
+TEST(SM75_Epilogue_with_reduction_threadblock, f16_tensor_op_128x64_64x32x8) {
+
+  //
+  // Define the warp-level matrix multiply
+  //
+
+  using ElementOutput = cutlass::half_t;
+  using ElementAccumulator = float;
+  using ElementCompute = float;
+  int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
+  int const kPartitionsK = 1;
+  
+  using Shape = cutlass::gemm::GemmShape<128, 64, 8>;
+  using WarpShape = cutlass::gemm::GemmShape<64, 32, 8>;
+  using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
+  using Element = cutlass::half_t;
+  using ElementC = ElementAccumulator;
+  using LayoutA = cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutB = cutlass::layout::RowMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutC = cutlass::layout::RowMajor;
+
+  using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
+      WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
+      LayoutC>::Type;
+
+  //
+  // Output operator
+  //
+
+  using OutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
+    ElementAccumulator,
+    ElementAccumulator,
+    ElementOutput,
+    ElementOutput,
+    kElementsPerAccess
+  >;
+
+  using ReductionOp = cutlass::plus<ElementAccumulator>;
+
+  //
+  // Define the epilogue
+  //
+
+  using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueWithReductionTensorOp<
+    Shape,
+    WarpMmaTensorOp,
+    kPartitionsK,
+    ElementOutput,
+    OutputOp,
+    ReductionOp,
+    kElementsPerAccess
+  >::Epilogue;
+
+  //
+  // Instantiate epilogue
+  //
+
+  EpilogueWithReductionTestbed<Epilogue> testbed;
+
+  bool passed = testbed.run_all();
+
+  EXPECT_TRUE(passed);
+}
+#endif
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM75_Epilogue_with_reduction_threadblock, f32_tensor_op_64x128_32x64x8) {
+
+  //
+  // Define the warp-level matrix multiply
+  //
+
+  using ElementOutput = float;
+  using ElementAccumulator = float;
+  using ElementCompute = float;
+  int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
+  int const kPartitionsK = 1;
+  
+  using Shape = cutlass::gemm::GemmShape<64, 128, 8>;
+  using WarpShape = cutlass::gemm::GemmShape<32, 64, 8>;
+  using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
+  using Element = cutlass::half_t;
+  using ElementC = ElementAccumulator;
+  using LayoutA = cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutB = cutlass::layout::RowMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutC = cutlass::layout::RowMajor;
+
+  using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
+      WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
+      LayoutC>::Type;
+
+  //
+  // Output operator
+  //
+
+  using OutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
+    ElementAccumulator,
+    ElementAccumulator,
+    ElementOutput,
+    ElementOutput,
+    kElementsPerAccess
+  >;
+
+  using ReductionOp = cutlass::plus<ElementAccumulator>;
+
+  //
+  // Define the epilogue
+  //
+
+  using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueWithReductionTensorOp<
+    Shape,
+    WarpMmaTensorOp,
+    kPartitionsK,
+    ElementOutput,
+    OutputOp,
+    ReductionOp,
+    kElementsPerAccess
+  >::Epilogue;
+
+  //
+  // Instantiate epilogue
+  //
+
+  EpilogueWithReductionTestbed<Epilogue> testbed;
+
+  bool passed = testbed.run_all();
+
+  EXPECT_TRUE(passed);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM75_Epilogue_with_reduction_threadblock, f16_tensor_op_64x128_32x64x8) {
+
+  //
+  // Define the warp-level matrix multiply
+  //
+
+  using ElementOutput = cutlass::half_t;
+  using ElementAccumulator = float;
+  using ElementCompute = float;
+  int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
+  int const kPartitionsK = 1;
+  
+  using Shape = cutlass::gemm::GemmShape<64, 128, 8>;
+  using WarpShape = cutlass::gemm::GemmShape<32, 64, 8>;
+  using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
+  using Element = cutlass::half_t;
+  using ElementC = ElementAccumulator;
+  using LayoutA = cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutB = cutlass::layout::RowMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutC = cutlass::layout::RowMajor;
+
+  using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
+      WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
+      LayoutC>::Type;
+
+  //
+  // Output operator
+  //
+
+  using OutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
+    ElementAccumulator,
+    ElementAccumulator,
+    ElementOutput,
+    ElementOutput,
+    kElementsPerAccess
+  >;
+
+  using ReductionOp = cutlass::plus<ElementAccumulator>;
+
+  //
+  // Define the epilogue
+  //
+
+  using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueWithReductionTensorOp<
+    Shape,
+    WarpMmaTensorOp,
+    kPartitionsK,
+    ElementOutput,
+    OutputOp,
+    ReductionOp,
+    kElementsPerAccess
+  >::Epilogue;
+
+  //
+  // Instantiate epilogue
+  //
+
+  EpilogueWithReductionTestbed<Epilogue> testbed;
+
+  bool passed = testbed.run_all();
+
+  EXPECT_TRUE(passed);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM75_Epilogue_with_reduction_threadblock, f32_tensor_op_128x256_64x64x8) {
+
+  //
+  // Define the warp-level matrix multiply
+  //
+
+  using ElementOutput = float;
+  using ElementAccumulator = float;
+  using ElementCompute = float;
+  int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
+  int const kPartitionsK = 1;
+  
+  using Shape = cutlass::gemm::GemmShape<128, 256, 8>;
+  using WarpShape = cutlass::gemm::GemmShape<64, 64, 8>;
+  using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
+  using Element = cutlass::half_t;
+  using ElementC = ElementAccumulator;
+  using LayoutA = cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutB = cutlass::layout::RowMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutC = cutlass::layout::RowMajor;
+
+  using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
+      WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
+      LayoutC>::Type;
+
+  //
+  // Output operator
+  //
+
+  using OutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
+    ElementAccumulator,
+    ElementAccumulator,
+    ElementOutput,
+    ElementOutput,
+    kElementsPerAccess
+  >;
+
+  using ReductionOp = cutlass::plus<ElementAccumulator>;
+
+  //
+  // Define the epilogue
+  //
+
+  using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueWithReductionTensorOp<
+    Shape,
+    WarpMmaTensorOp,
+    kPartitionsK,
+    ElementOutput,
+    OutputOp,
+    ReductionOp,
+    kElementsPerAccess
+  >::Epilogue;
+
+  //
+  // Instantiate epilogue
+  //
+
+  EpilogueWithReductionTestbed<Epilogue> testbed;
+
+  bool passed = testbed.run_all();
+
+  EXPECT_TRUE(passed);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM75_Epilogue_with_reduction_threadblock, f16_tensor_op_128x256_64x64x8) {
+
+  //
+  // Define the warp-level matrix multiply
+  //
+
+  using ElementOutput = cutlass::half_t;
+  using ElementAccumulator = float;
+  using ElementCompute = float;
+  int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
+  int const kPartitionsK = 1;
+  
+  using Shape = cutlass::gemm::GemmShape<128, 256, 8>;
+  using WarpShape = cutlass::gemm::GemmShape<64, 64, 8>;
+  using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
+  using Element = cutlass::half_t;
+  using ElementC = ElementAccumulator;
+  using LayoutA = cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutB = cutlass::layout::RowMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutC = cutlass::layout::RowMajor;
+
+  using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
+      WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
+      LayoutC>::Type;
+
+  //
+  // Output operator
+  //
+
+  using OutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
+    ElementAccumulator,
+    ElementAccumulator,
+    ElementOutput,
+    ElementOutput,
+    kElementsPerAccess
+  >;
+
+  using ReductionOp = cutlass::plus<ElementAccumulator>;
+
+  //
+  // Define the epilogue
+  //
+
+  using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueWithReductionTensorOp<
+    Shape,
+    WarpMmaTensorOp,
+    kPartitionsK,
+    ElementOutput,
+    OutputOp,
+    ReductionOp,
+    kElementsPerAccess
+  >::Epilogue;
+
+  //
+  // Instantiate epilogue
+  //
+
+  EpilogueWithReductionTestbed<Epilogue> testbed;
+
+  bool passed = testbed.run_all();
+
+  EXPECT_TRUE(passed);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM75_Epilogue_with_reduction_threadblock, f32_tensor_op_256x128_64x64x8) {
+
+  //
+  // Define the warp-level matrix multiply
+  //
+
+  using ElementOutput = float;
+  using ElementAccumulator = float;
+  using ElementCompute = float;
+  int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
+  int const kPartitionsK = 1;
+  
+  using Shape = cutlass::gemm::GemmShape<256, 128, 8>;
+  using WarpShape = cutlass::gemm::GemmShape<64, 64, 8>;
+  using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
+  using Element = cutlass::half_t;
+  using ElementC = ElementAccumulator;
+  using LayoutA = cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutB = cutlass::layout::RowMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutC = cutlass::layout::RowMajor;
+
+  using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
+      WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
+      LayoutC>::Type;
+
+  //
+  // Output operator
+  //
+
+  using OutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
+    ElementAccumulator,
+    ElementAccumulator,
+    ElementOutput,
+    ElementOutput,
+    kElementsPerAccess
+  >;
+
+  using ReductionOp = cutlass::plus<ElementAccumulator>;
+
+  //
+  // Define the epilogue
+  //
+
+  using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueWithReductionTensorOp<
+    Shape,
+    WarpMmaTensorOp,
+    kPartitionsK,
+    ElementOutput,
+    OutputOp,
+    ReductionOp,
+    kElementsPerAccess
+  >::Epilogue;
+
+  //
+  // Instantiate epilogue
+  //
+
+  EpilogueWithReductionTestbed<Epilogue> testbed;
+
+  bool passed = testbed.run_all();
+
+  EXPECT_TRUE(passed);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM75_Epilogue_with_reduction_threadblock, f16_tensor_op_256x128_64x64x8) {
+
+  //
+  // Define the warp-level matrix multiply
+  //
+
+  using ElementOutput = cutlass::half_t;
+  using ElementAccumulator = float;
+  using ElementCompute = float;
+  int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
+  int const kPartitionsK = 1;
+  
+  using Shape = cutlass::gemm::GemmShape<256, 128, 8>;
+  using WarpShape = cutlass::gemm::GemmShape<64, 64, 8>;
+  using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
+  using Element = cutlass::half_t;
+  using ElementC = ElementAccumulator;
+  using LayoutA = cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutB = cutlass::layout::RowMajorTensorOpMultiplicandCongruous<
+      cutlass::sizeof_bits<Element>::value, 64>;
+  using LayoutC = cutlass::layout::RowMajor;
+
+  using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
+      WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
+      LayoutC>::Type;
+
+  //
+  // Output operator
+  //
+
+  using OutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
+    ElementAccumulator,
+    ElementAccumulator,
+    ElementOutput,
+    ElementOutput,
+    kElementsPerAccess
+  >;
+
+  using ReductionOp = cutlass::plus<ElementAccumulator>;
+
+  //
+  // Define the epilogue
+  //
+
+  using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueWithReductionTensorOp<
+    Shape,
+    WarpMmaTensorOp,
+    kPartitionsK,
+    ElementOutput,
+    OutputOp,
+    ReductionOp,
+    kElementsPerAccess
+  >::Epilogue;
+
+  //
+  // Instantiate epilogue
+  //
+
+  EpilogueWithReductionTestbed<Epilogue> testbed;
+
+  bool passed = testbed.run_all();
+
+  EXPECT_TRUE(passed);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////

test/unit/epilogue/threadblock/epilogue_with_reduction_testbed.h

@@ -0,0 +1,429 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+/*! \file
+  
+    \brief Unit tests for epilogues
+*/
+#pragma once
+
+#include <fstream>
+
+#include "../../common/cutlass_unit_test.h"
+
+#include "cutlass/aligned_buffer.h"
+#include "cutlass/half.h"
+#include "cutlass/complex.h"
+
+#include "cutlass/epilogue/thread/linear_combination.h"
+
+#include "cutlass/util/host_tensor.h"
+#include "cutlass/util/tensor_view_io.h"
+#include "cutlass/util/reference/host/tensor_fill.h"
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace test {
+namespace kernel {
+
+template <typename Epilogue>
+__global__ void epilogue_with_reduction_threadblock(
+  typename Epilogue::ElementVector *ptr_Reduction,
+  typename Epilogue::OutputTileIterator::Params params_D,
+  typename Epilogue::OutputTileIterator::Element *ptr_D,
+  typename Epilogue::OutputTileIterator::Params params_C,
+  typename Epilogue::OutputTileIterator::Element *ptr_C,
+  typename Epilogue::TensorTileIterator::Params params_Tensor,
+  typename Epilogue::TensorTileIterator::Element *ptr_Tensor,
+  typename Epilogue::OutputOp::Params params_output_op,
+  cutlass::MatrixCoord problem_size,
+  cutlass::TensorRef<
+    typename Epilogue::WarpMmaOperator::ElementC, 
+    typename Epilogue::WarpMmaOperator::LayoutC> accumulator_ref,
+  int epilogue_count = 1) {
+
+  __shared__ typename Epilogue::SharedStorage shared_storage;
+
+  int thread_idx = threadIdx.x;
+  int warp_idx = threadIdx.x / 32;
+  int lane_idx = threadIdx.x % 32;
+
+  //
+  // Construct the epilogue
+  //
+
+  // Tile iterator writing to output tile
+  typename Epilogue::OutputTileIterator iterator_D(
+    params_D,
+    ptr_D,
+    problem_size,
+    thread_idx
+  );
+
+  // Tile iterator writing to output tile
+  typename Epilogue::OutputTileIterator iterator_C(
+    params_C,
+    ptr_C,
+    problem_size,
+    thread_idx
+  );
+
+  // Tile iterator writing to output tile
+  typename Epilogue::TensorTileIterator iterator_T(
+    params_Tensor,
+    ptr_Tensor,
+    problem_size,
+    thread_idx
+  );
+
+  // Epilogue operator
+  Epilogue epilogue(
+    shared_storage, 
+    thread_idx, 
+    warp_idx, 
+    lane_idx);
+
+  //
+  // Initialize the accumulators
+  //
+
+  int warp_mn = warp_idx % (Epilogue::WarpCount::kM * Epilogue::WarpCount::kN);
+  int warp_m = warp_mn % Epilogue::WarpCount::kM;
+  int warp_n = warp_mn / Epilogue::WarpCount::kM;
+
+  accumulator_ref.add_coord_offset({
+    warp_m * Epilogue::WarpMmaOperator::Shape::kM, 
+    warp_n * Epilogue::WarpMmaOperator::Shape::kN});
+
+  typename Epilogue::WarpMmaOperator::IteratorC accumulator_iterator(accumulator_ref, lane_idx);
+  
+  typename Epilogue::AccumulatorTile accumulators;
+
+  accumulators.clear();
+  accumulator_iterator.load(accumulators);
+
+#if 0
+  // For debugging, enable this block of code to fill each accumulator element with its
+  // source thread ID.
+  CUTLASS_PRAGMA_UNROLL
+  for (int i = 0; i < accumulators.size(); ++i) {
+    typename Epilogue::WarpMmaOperator::ElementC x(threadIdx.x);
+    //typename Epilogue::WarpMmaOperator::ElementC x(i);
+    accumulators[i] = x;
+  }
+
+  /*
+  #pragma unroll 1
+  for (int tid = 0; tid < 32; ++tid) {
+    if (tid == thread_idx) {
+      printf("\nT%d: ", thread_idx);
+      CUTLASS_PRAGMA_UNROLL
+      for (int i = 0; i < accumulators.size(); ++i) {
+        printf("%d ", int(accumulators[i]));
+      }  
+    }
+  }
+
+  if (thread_idx == 0) {
+    printf("\n\n");  
+  }
+  */
+
+  __syncthreads();
+
+#endif
+
+  //
+  // Perform the epilogue operation
+  //
+
+  typename Epilogue::OutputOp output_op(params_output_op);
+
+  // Place the epilogue in a loop
+  for (int iter = 0; iter < epilogue_count; ++iter) {
+    epilogue(output_op, ptr_Reduction, iterator_D, accumulators, iterator_C, iterator_T);
+  }
+}
+
+} // namespace kernel
+} // namespace test
+
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <
+  typename Epilogue_
+>
+class EpilogueWithReductionTestbed {
+public:
+
+  using Epilogue = Epilogue_;
+  using ElementAccumulator = typename Epilogue::ElementAccumulator;
+  using ElementCompute = typename Epilogue::OutputOp::ElementCompute;
+  using ElementTensor = typename Epilogue::TensorTileIterator::Element;
+  using ElementOutput = typename Epilogue::ElementOutput;
+  using OutputOpParams = typename Epilogue::OutputOp::Params;
+
+public:
+
+  //
+  // Data members
+  //
+
+  cutlass::MatrixCoord quantized_size;
+  cutlass::HostTensor<ElementAccumulator, cutlass::layout::RowMajor> accumulator_tensor;
+  cutlass::HostTensor<ElementOutput, cutlass::layout::RowMajor> source_tensor;
+  cutlass::HostTensor<ElementOutput, cutlass::layout::RowMajor> output_tensor;
+  cutlass::HostTensor<ElementTensor, cutlass::layout::RowMajor> additional_tensor;
+  cutlass::HostTensor<ElementAccumulator, cutlass::layout::RowMajor> reduction_tensor;
+
+
+public:
+
+  //
+  // Methods
+  //
+
+  EpilogueWithReductionTestbed(): 
+    quantized_size(Epilogue::Shape::kM, Epilogue::Shape::kN),
+    accumulator_tensor({Epilogue::Shape::kM, Epilogue::Shape::kN}),
+    source_tensor({Epilogue::Shape::kM, Epilogue::Shape::kN}),
+    output_tensor({Epilogue::Shape::kM, Epilogue::Shape::kN}),
+    additional_tensor({Epilogue::Shape::kM, Epilogue::Shape::kN}),
+    reduction_tensor({1, Epilogue::Shape::kN}) {
+
+    //
+    // Initialize problem space
+    //
+
+    uint64_t seed = 2019;
+
+    cutlass::reference::host::TensorFillRandomUniform(
+      accumulator_tensor.host_view(), 
+      seed, 
+      20, 
+      -20, 
+      0);
+
+    cutlass::reference::host::TensorFillRandomUniform(
+      source_tensor.host_view(),
+      seed + 2018, 
+      20, 
+      -20, 
+      0);
+
+    cutlass::reference::host::TensorFill(additional_tensor.host_view(), ElementTensor(1));
+  }
+
+  bool run_all() {
+   
+    /*
+    double alpha_values[] = {1, 0, 2.25};
+    double beta_values[] = {0, 1, -1.25};
+
+    // Test runtime explodes if we tried to test every case exhaustively. This tests the full
+    // output tile and several smaller sizes to stress predication.
+    for (int m_idx = 0; m_idx < 3; ++m_idx) {
+      for (int n_idx = 0; n_idx < 3; ++n_idx) {
+
+        int m = quantized_size.row() - m_idx * 3;
+        int n = quantized_size.column() - n_idx * Epilogue::kElementsPerAccess;
+
+        for (double const &alpha : alpha_values) {
+          for (double const &beta : beta_values) {
+
+            bool passed = run({m, n}, {cutlass::from_real<ElementCompute>(alpha), cutlass::from_real<ElementCompute>(beta)});
+
+            if (!passed) {
+              return false;
+            }
+          }
+        }
+      }
+    }
+    return true;
+    */
+
+    double alpha = 1;
+    double beta = 0;
+
+    return run(
+      {quantized_size.row(), quantized_size.column()},
+      {cutlass::from_real<ElementCompute>(alpha), cutlass::from_real<ElementCompute>(beta)});
+  }
+
+  /// Runs the test
+  bool run(
+    cutlass::MatrixCoord problem_size,
+    OutputOpParams output_params) { 
+
+    //
+    // Initialize problem space
+    //
+
+    ElementOutput default_output = ElementOutput(-127);
+    ElementAccumulator default_reduction = ElementAccumulator();
+
+    cutlass::reference::host::TensorFill(output_tensor.host_view(), default_output);
+    cutlass::reference::host::TensorFill(reduction_tensor.host_view(), default_reduction);
+
+    accumulator_tensor.sync_device();
+    output_tensor.sync_device();
+    source_tensor.sync_device();
+    additional_tensor.sync_device();
+    reduction_tensor.sync_device();
+
+    //
+    // Initialize epilogue parameters
+    //
+
+    typename Epilogue::OutputTileIterator::Params params_D(output_tensor.device_ref().layout());
+    typename Epilogue::OutputTileIterator::Params params_C(source_tensor.device_ref().layout());
+    typename Epilogue::TensorTileIterator::Params params_T(additional_tensor.device_ref().layout());
+
+    //
+    // Launch kernel
+    //
+
+    dim3 grid(1, 1);
+    dim3 block(Epilogue::WarpCount::kCount * 32, 1);
+
+    test::kernel::epilogue_with_reduction_threadblock<Epilogue><<< grid, block >>>(
+      reduction_tensor.device_data(),
+      params_D,
+      output_tensor.device_data(),
+      params_C,
+      source_tensor.device_data(),
+      params_T,
+      additional_tensor.device_data(),
+      output_params,
+      problem_size, 
+      accumulator_tensor.device_view());
+
+    cudaError_t result = cudaDeviceSynchronize();
+
+    if (result != cudaSuccess) {
+      std::cerr << "Kernel error: " << cudaGetErrorString(result) << std::endl;
+      return false;
+    }
+
+    //
+    // Verify results
+    //
+    output_tensor.sync_host();
+    reduction_tensor.sync_host();
+
+    int errors = 0;
+    int const kMaxErrors = 5;
+
+    //
+    // The output has two parts:
+    //  - GEMM tensor epilogue in canonical layout
+    //  - partial reduction in canonical row-major layout
+    //
+
+    // Verify the GEMM tensor output
+    for (int r = 0; errors < kMaxErrors && r < quantized_size.row(); ++r) {
+      for (int c = 0; errors < kMaxErrors && c < quantized_size.column(); ++c) {
+
+        cutlass::MatrixCoord coord{r, c};
+        ElementOutput got = output_tensor.at(coord);
+        
+        ElementOutput expected;
+        if (coord.row() < problem_size.row() && coord.column() < problem_size.column()) {
+
+          expected = ElementOutput(output_params.alpha * ElementCompute(accumulator_tensor.at(coord)) + 
+            output_params.beta * ElementCompute(source_tensor.at(coord)));
+        }
+        else {
+          expected = default_output;
+        }
+
+        if (expected != got) {
+
+          using OutputIO = cutlass::ScalarIO<ElementOutput>;
+
+          EXPECT_TRUE(false)
+            << "-------\n"
+            << "Error - output element (" << coord << ") - expected: " 
+            << OutputIO(expected) 
+            << ",  got: " << OutputIO(got) << std::endl;
+
+          ++errors;
+        }
+      }
+    }
+
+    // Verify the partial reduction
+    for (int c = 0; c < quantized_size.column(); ++c) {
+
+      ElementAccumulator reduction_acc = ElementAccumulator();
+
+      for (int r = 0; r < quantized_size.row(); ++r) {
+        reduction_acc += accumulator_tensor.at({r, c});
+      }
+
+      ElementAccumulator expected = default_reduction;
+      ElementAccumulator got = reduction_tensor.at({0, c});
+
+      if (c < problem_size.column()) {
+        expected = reduction_acc;
+      }
+      else {
+        expected = default_reduction;
+      }
+
+      if (expected != got) {
+        
+        using OutputIO = cutlass::ScalarIO<ElementAccumulator>;
+
+        EXPECT_TRUE(false)
+          << "-------\n"
+          << "Error - reduction element (" << c << ") - expected: " 
+          << OutputIO(expected) 
+          << ", got: " << OutputIO(got) << std::endl;
+      }
+    }
+
+    //
+    // Report results on error
+    //
+
+    if (errors) {
+      std::stringstream ss;
+      ss 
+        << "output_tensor_op_" << Epilogue::Shape::kM << "x" << Epilogue::Shape::kN << "_" 
+        << Epilogue::WarpTileIterator::WarpShape::kM << "x" 
+        << Epilogue::WarpTileIterator::WarpShape::kN 
+        << "_slice_" << Epilogue::WarpCount::kK << ".csv"; 
+
+      std::ofstream output_file(ss.str()); 
+      output_file << output_tensor.host_view(); 
+    }
+
+    return !errors;
+  }
+};
+
+/////////////////////////////////////////////////////////////////////////////////////////////////

test/unit/epilogue/threadblock/output_tile_threadmap.cu

@@ -63,7 +63,7 @@ struct OutputTileThreadMapExpr {
   };
 
   int const kWarpSize = 32;
-  int const kMemoryAccessSize = 128;  // size in bytes of the preferred memory access size
+  int const kMemoryAccessSize = 256;  // size in bytes of the preferred memory access size
 
   //
   // Data members

test/unit/epilogue/threadblock/testbed.h

@@ -28,13 +28,14 @@
 #pragma once
 
 #include <fstream>
+#include <cfenv>
 
 #include "../../common/cutlass_unit_test.h"
 
 #include "cutlass/aligned_buffer.h"
 #include "cutlass/half.h"
 #include "cutlass/complex.h"
-
+#include "cutlass/quaternion.h"
 #include "cutlass/epilogue/thread/linear_combination.h"
 
 #include "cutlass/util/host_tensor.h"
@@ -307,10 +308,18 @@ public:
         
         ElementOutput expected;
         if (coord.row() < problem_size.row() && coord.column() < problem_size.column()) {
-          expected = ElementOutput(output_params.alpha * ElementCompute(accumulator_tensor.at(coord)) + 
-            output_params.beta * ElementCompute(source_tensor.at(coord)));
-        }
-        else {
+          ElementCompute intermediate =
+            output_params.alpha * ElementCompute(accumulator_tensor.at(coord)) + 
+            output_params.beta * ElementCompute(source_tensor.at(coord));
+          
+          if (std::numeric_limits<ElementOutput>::is_integer
+              && !std::numeric_limits<ElementCompute>::is_integer) {
+            std::fesetround(FE_TONEAREST);
+            expected = ElementOutput(std::nearbyint(float(cutlass::real(intermediate))));
+          } else {
+            expected = ElementOutput(intermediate);
+          }
+        } else {
           expected = default_output;
         }
 
@@ -322,7 +331,11 @@ public:
             << "-------\n"
             << "Error - output element (" << coord << ") - expected: " 
             << OutputIO(expected) 
-            << ",  got: " << OutputIO(got) << std::endl;
+            << ",  got: " << OutputIO(got)
+            << ",  accum: " << (accumulator_tensor.at(coord))
+            << ",  source: " << OutputIO(source_tensor.at(coord))
+            << ",  alpha: " << (output_params.alpha)
+            << ",  beta: " << (output_params.beta) << "\n";
 
           ++errors;
         }

test/unit/gemm/device/CMakeLists.txt

@@ -34,6 +34,7 @@ add_custom_target(
   cutlass_test_unit_gemm_device_wmma
   cutlass_test_unit_gemm_device_tensorop_planar_complex
   cutlass_test_unit_gemm_device_sparse_tensorop_sm80
+  cutlass_test_unit_gemv_device
 )
 
 add_custom_target(
@@ -50,6 +51,7 @@ add_custom_target(
   test_unit_gemm_device_wmma
   test_unit_gemm_device_tensorop_planar_complex
   test_unit_gemm_device_sparse_tensorop_sm80
+  test_unit_gemv_device
 )
 
 cutlass_test_unit_add_executable(
@@ -66,6 +68,11 @@ cutlass_test_unit_add_executable(
   simt_cgemm_tn_sm50.cu
   simt_cgemm_tt_sm50.cu
 
+  simt_qgemm_nn_sm50.cu
+  simt_qgemm_nt_sm50.cu
+  simt_qgemm_tn_sm50.cu
+  simt_qgemm_tt_sm50.cu
+
   simt_dgemm_nn_sm50.cu
   simt_dgemm_nt_sm50.cu
   simt_dgemm_tn_sm50.cu
@@ -203,6 +210,7 @@ cutlass_test_unit_add_executable(
 
   gemm_f32n_f32n_f32t_tensor_op_f32_sm80.cu
   gemm_f32n_f32n_f32t_tensor_op_bf16_f32_sm80.cu
+
 )
 
 cutlass_test_unit_add_executable(
@@ -332,3 +340,36 @@ cutlass_test_unit_add_executable(
   gemm_s4t_s4n_s32t_tensor_op_s32_sparse_sm80.cu
 )
 
+
+cutlass_test_unit_add_executable(
+  cutlass_test_unit_gemv_device
+
+  BATCH_SOURCES ON
+  BATCH_SIZE 4
+
+  gemv.cu
+)
+
+if (NOT CUDA_COMPILER MATCHES "[Cc]lang")
+
+add_dependencies(
+  cutlass_test_unit_gemm_device
+  cutlass_test_unit_gemm_device_gemm_with_fused_epilogue_tensorop
+  )
+
+add_dependencies(
+  test_unit_gemm_device
+  test_unit_gemm_device_gemm_with_fused_epilogue_tensorop
+  )
+
+cutlass_test_unit_add_executable(
+  cutlass_test_unit_gemm_device_gemm_with_fused_epilogue_tensorop
+  
+  gemm_with_reduction_f16n_f16n_f16n_tensorop_f32_sm75.cu
+  gemm_with_broadcast_f16n_f16n_f16n_tensorop_f32_sm75.cu
+
+  gemm_with_reduction_f16t_f16n_f16n_tensorop_f32_sm80.cu
+)
+
+endif()
+

test/unit/gemm/device/gemm_f64n_f64t_f64t_tensor_op_f64_sm80.cu

@@ -41,7 +41,6 @@
 #include "testbed.h"
 
 #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
-
 /////////////////////////////////////////////////////////////////////////////////////////////////
 
 TEST(SM80_Device_Gemm_f64n_f64t_f64t_tensor_op_f64, 32x32x16_16x16x16) {
@@ -209,4 +208,45 @@ TEST(SM80_Device_Gemm_f64n_f64t_f64t_tensor_op_f64, 128x128x16_32x64x16) {
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
 
+TEST(SM80_Device_Gemm_f64an_f64at_f64at_tensor_op_f64, 128x128x16_32x64x16) {
+
+  using ElementOutput = double;
+  using ElementAccumulator = double;
+
+  using LayoutA = cutlass::layout::AffineRank2ColumnMajor;
+  using LayoutB = cutlass::layout::AffineRank2RowMajor;
+  using LayoutC = cutlass::layout::AffineRankN<2>;
+
+  using Gemm = cutlass::gemm::device::Gemm<
+    double,
+    LayoutA,
+    double,
+    LayoutB,
+    ElementOutput,
+    LayoutC, 
+    ElementAccumulator,
+    cutlass::arch::OpClassTensorOp,
+    cutlass::arch::Sm80,
+    cutlass::gemm::GemmShape<128, 128, 16>,
+    cutlass::gemm::GemmShape<32, 64, 16>,
+    cutlass::gemm::GemmShape<8, 8, 4>,
+    cutlass::epilogue::thread::LinearCombination<
+      ElementOutput,
+      1,
+      ElementAccumulator,
+      ElementAccumulator
+    >,
+    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+    3
+  >;
+
+  typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
+  typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
+  typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
+
+  EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
 #endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)

test/unit/gemm/device/gemm_f64t_f64n_f64t_tensor_op_f64_sm80.cu

@@ -209,4 +209,45 @@ TEST(SM80_Device_Gemm_f64t_f64n_f64t_tensor_op_f64, 128x128x16_32x64x16) {
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
 
+TEST(SM80_Device_Gemm_f64at_f64an_f64at_tensor_op_f64, 128x128x16_32x64x16) {
+
+  using ElementOutput = double;
+  using ElementAccumulator = double;
+
+  using LayoutA = cutlass::layout::AffineRank2RowMajor;
+  using LayoutB = cutlass::layout::AffineRank2ColumnMajor;
+  using LayoutC = cutlass::layout::AffineRankN<2>;
+
+  using Gemm = cutlass::gemm::device::Gemm<
+    double,
+    LayoutA,
+    double,
+    LayoutB,
+    ElementOutput,
+    LayoutC, 
+    ElementAccumulator,
+    cutlass::arch::OpClassTensorOp,
+    cutlass::arch::Sm80,
+    cutlass::gemm::GemmShape<128, 128, 16>,
+    cutlass::gemm::GemmShape<32, 64, 16>,
+    cutlass::gemm::GemmShape<8, 8, 4>,
+    cutlass::epilogue::thread::LinearCombination<
+      ElementOutput,
+      1,
+      ElementAccumulator,
+      ElementAccumulator
+    >,
+    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+    3
+  >;
+
+  typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
+  typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
+  typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
+
+  EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
 #endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)

test/unit/gemm/device/gemm_planar_complex_f16_f16_f32_tensor_op_sm70.cu

@@ -126,6 +126,222 @@ TEST(SM70_Device_GemmPlanarComplex_f16n_f16t_f32n_tensor_op_f32_884, 64x64x32_32
 }
 
 
+////////////////////////////////////////////////////////////////////////////////
+
+using gemm_planar_complex_s884_nn_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
+  cutlass::half_t,
+  cutlass::layout::ColumnMajor,
+  cutlass::ComplexTransform::kNone,
+  8,
+  cutlass::half_t,
+  cutlass::layout::ColumnMajor,
+  cutlass::ComplexTransform::kNone,
+  8,
+  float,
+  cutlass::layout::RowMajor,
+  float,
+  cutlass::arch::OpClassTensorOp,
+  cutlass::arch::Sm70,
+  cutlass::gemm::GemmShape<128, 64, 32>,
+  cutlass::gemm::GemmShape<32, 32, 32>,
+  cutlass::gemm::GemmShape<8, 8, 4>,
+  cutlass::epilogue::thread::LinearCombinationPlanarComplex<
+    float,
+    4,
+    float,
+    float
+  >,
+  cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+  2,
+  cutlass::arch::OpMultiplyAdd
+>::GemmKernel;
+
+struct gemm_planar_complex_s884_nn : gemm_planar_complex_s884_nn_base {
+
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM70_Device_GemmPlanarComplex_f16n_f16n_f32n_tensor_op_f32_884, 128x64x32_32x32x32) {
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_s884_nn>;
+
+  EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+using gemm_planar_complex_f16_s884_f16_nn_128x64_32x2_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
+  cutlass::half_t,
+  cutlass::layout::ColumnMajor,
+  cutlass::ComplexTransform::kNone,
+  8,
+  cutlass::half_t,
+  cutlass::layout::ColumnMajor,
+  cutlass::ComplexTransform::kNone,
+  8,
+  cutlass::half_t,
+  cutlass::layout::RowMajor,
+  float,
+  cutlass::arch::OpClassTensorOp,
+  cutlass::arch::Sm70,
+  cutlass::gemm::GemmShape<128, 64, 32>,
+  cutlass::gemm::GemmShape<32, 32, 32>,
+  cutlass::gemm::GemmShape<8, 8, 4>,
+  cutlass::epilogue::thread::LinearCombinationPlanarComplex<
+    cutlass::half_t,
+    8,
+    float,
+    float
+  >,
+  cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+  2,
+  cutlass::arch::OpMultiplyAdd
+>::GemmKernel;
+
+struct gemm_planar_complex_f16_s884_f16_nn_128x64_32x2 : gemm_planar_complex_f16_s884_f16_nn_128x64_32x2_base {
+
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM70_Device_GemmPlanarComplex_f16n_f16n_f16n_tensor_op_f32_884, 128x64x32_32x32x32) {
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_f16_s884_f16_nn_128x64_32x2>;
+
+  EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+using gemm_planar_complex_f16_s884_f16_nn_64x128_32x2_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
+  cutlass::half_t,
+  cutlass::layout::ColumnMajor,
+  cutlass::ComplexTransform::kNone,
+  8,
+  cutlass::half_t,
+  cutlass::layout::ColumnMajor,
+  cutlass::ComplexTransform::kNone,
+  8,
+  cutlass::half_t,
+  cutlass::layout::RowMajor,
+  float,
+  cutlass::arch::OpClassTensorOp,
+  cutlass::arch::Sm70,
+  cutlass::gemm::GemmShape<64, 128, 32>,
+  cutlass::gemm::GemmShape<32, 32, 32>,
+  cutlass::gemm::GemmShape<8, 8, 4>,
+  cutlass::epilogue::thread::LinearCombinationPlanarComplex<
+    cutlass::half_t,
+    8,
+    float,
+    float
+  >,
+  cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+  2,
+  cutlass::arch::OpMultiplyAdd
+>::GemmKernel;
+
+struct gemm_planar_complex_f16_s884_f16_nn_64x128_32x2 : gemm_planar_complex_f16_s884_f16_nn_64x128_32x2_base {
+
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM70_Device_GemmPlanarComplex_f16n_f16n_f16n_tensor_op_f32_884, 64x128x32_32x32x32) {
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_f16_s884_f16_nn_64x128_32x2>;
+
+  EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
+}
+
+
+////////////////////////////////////////////////////////////////////////////////
+
+using gemm_planar_complex_f16_s884_f16_tt_128x64_32x2_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
+  cutlass::half_t,
+  cutlass::layout::RowMajor,
+  cutlass::ComplexTransform::kNone,
+  8,
+  cutlass::half_t,
+  cutlass::layout::RowMajor,
+  cutlass::ComplexTransform::kNone,
+  8,
+  cutlass::half_t,
+  cutlass::layout::RowMajor,
+  float,
+  cutlass::arch::OpClassTensorOp,
+  cutlass::arch::Sm70,
+  cutlass::gemm::GemmShape<128, 64, 32>,
+  cutlass::gemm::GemmShape<32, 32, 32>,
+  cutlass::gemm::GemmShape<8, 8, 4>,
+  cutlass::epilogue::thread::LinearCombinationPlanarComplex<
+    cutlass::half_t,
+    8,
+    float,
+    float
+  >,
+  cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+  2,
+  cutlass::arch::OpMultiplyAdd
+>::GemmKernel;
+
+struct gemm_planar_complex_f16_s884_f16_tt_128x64_32x2 : gemm_planar_complex_f16_s884_f16_tt_128x64_32x2_base {
+
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM70_Device_GemmPlanarComplex_f16t_f16t_f16n_tensor_op_f32_884, 128x64x32_32x32x32) {
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_f16_s884_f16_tt_128x64_32x2>;
+
+  EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+using gemm_planar_complex_f16_s884_f16_tt_64x128_32x2_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
+  cutlass::half_t,
+  cutlass::layout::RowMajor,
+  cutlass::ComplexTransform::kNone,
+  8,
+  cutlass::half_t,
+  cutlass::layout::RowMajor,
+  cutlass::ComplexTransform::kNone,
+  8,
+  cutlass::half_t,
+  cutlass::layout::RowMajor,
+  float,
+  cutlass::arch::OpClassTensorOp,
+  cutlass::arch::Sm70,
+  cutlass::gemm::GemmShape<64, 128, 32>,
+  cutlass::gemm::GemmShape<32, 32, 32>,
+  cutlass::gemm::GemmShape<8, 8, 4>,
+  cutlass::epilogue::thread::LinearCombinationPlanarComplex<
+    cutlass::half_t,
+    8,
+    float,
+    float
+  >,
+  cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+  2,
+  cutlass::arch::OpMultiplyAdd
+>::GemmKernel;
+
+struct gemm_planar_complex_f16_s884_f16_tt_64x128_32x2 : gemm_planar_complex_f16_s884_f16_tt_64x128_32x2_base {
+
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM70_Device_GemmPlanarComplex_f16t_f16t_f16n_tensor_op_f32_884, 64x128x32_32x32x32) {
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_f16_s884_f16_tt_64x128_32x2>;
+
+  EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
 #endif // #if defined(CUTLASS_ARCH_MMA_SM70_SUPPORTED)

test/unit/gemm/device/gemm_planar_complex_f16_f16_f32_tensor_op_sm80.cu

@@ -81,6 +81,48 @@ TEST(SM80_Device_GemmPlanarComplex_f16t_f16n_f32n_tensor_op_f32_16816, 64x64x32_
   EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
 }
 
+////////////////////////////////////////////////////////////////////////////////
+
+using gemm_planar_complex_f16_s16816_tn_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
+  cutlass::half_t,
+  cutlass::layout::RowMajor,
+  cutlass::ComplexTransform::kNone,
+  8,
+  cutlass::half_t,
+  cutlass::layout::ColumnMajor,
+  cutlass::ComplexTransform::kNone,
+  8,
+  cutlass::half_t,
+  cutlass::layout::RowMajor,
+  float,
+  cutlass::arch::OpClassTensorOp,
+  cutlass::arch::Sm80,
+  cutlass::gemm::GemmShape<64, 64, 32>,
+  cutlass::gemm::GemmShape<32, 32, 32>,
+  cutlass::gemm::GemmShape<16, 8, 16>,
+  cutlass::epilogue::thread::LinearCombinationPlanarComplex<
+    float,
+    4,
+    float,
+    float
+  >,
+  cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+  3,
+  cutlass::arch::OpMultiplyAdd
+>::GemmKernel;
+
+struct gemm_planar_complex_f16_s16816_tn : gemm_planar_complex_f16_s16816_tn_base {
+
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM80_Device_GemmPlanarComplex_f16t_f16n_f16n_tensor_op_f32_16816, 64x64x32_32x32x32) {
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_f16_s16816_tn>;
+
+  EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
+}
 
 ////////////////////////////////////////////////////////////////////////////////
 
@@ -127,6 +169,49 @@ TEST(SM80_Device_GemmPlanarComplex_f16h_f16c_f32n_tensor_op_f32_16816, 64x64x32_
 
 ////////////////////////////////////////////////////////////////////////////////
 
+using gemm_planar_complex_f16_s16816_hc_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
+  cutlass::half_t,
+  cutlass::layout::RowMajor,
+  cutlass::ComplexTransform::kConjugate,
+  8,
+  cutlass::half_t,
+  cutlass::layout::ColumnMajor,
+  cutlass::ComplexTransform::kConjugate,
+  8,
+  cutlass::half_t,
+  cutlass::layout::RowMajor,
+  float,
+  cutlass::arch::OpClassTensorOp,
+  cutlass::arch::Sm80,
+  cutlass::gemm::GemmShape<64, 64, 32>,
+  cutlass::gemm::GemmShape<32, 32, 32>,
+  cutlass::gemm::GemmShape<16, 8, 16>,
+  cutlass::epilogue::thread::LinearCombinationPlanarComplex<
+    float,
+    4,
+    float,
+    float
+  >,
+  cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+  3,
+  cutlass::arch::OpMultiplyAdd
+>::GemmKernel;
+
+struct gemm_planar_complex_f16_s16816_hc : gemm_planar_complex_f16_s16816_hc_base {
+
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM80_Device_GemmPlanarComplex_f16h_f16c_f16n_tensor_op_f32_16816, 64x64x32_32x32x32) {
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_f16_s16816_hc>;
+
+  EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
 using gemm_planar_complex_s16816_nt_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
   cutlass::half_t,
   cutlass::layout::ColumnMajor,
@@ -168,6 +253,50 @@ TEST(SM80_Device_GemmPlanarComplex_f16n_f16t_f32n_tensor_op_f32_16816, 64x64x32_
   EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
 }
 
+
+////////////////////////////////////////////////////////////////////////////////
+
+using gemm_planar_complex_f16_s16816_nt_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
+  cutlass::half_t,
+  cutlass::layout::ColumnMajor,
+  cutlass::ComplexTransform::kNone,
+  8,
+  cutlass::half_t,
+  cutlass::layout::RowMajor,
+  cutlass::ComplexTransform::kNone,
+  8,
+  cutlass::half_t,
+  cutlass::layout::RowMajor,
+  float,
+  cutlass::arch::OpClassTensorOp,
+  cutlass::arch::Sm80,
+  cutlass::gemm::GemmShape<64, 64, 32>,
+  cutlass::gemm::GemmShape<32, 32, 32>,
+  cutlass::gemm::GemmShape<16, 8, 16>,
+  cutlass::epilogue::thread::LinearCombinationPlanarComplex<
+    float,
+    4,
+    float,
+    float
+  >,
+  cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+  3,
+  cutlass::arch::OpMultiplyAdd
+>::GemmKernel;
+
+struct gemm_planar_complex_f16_s16816_nt : gemm_planar_complex_f16_s16816_nt_base {
+
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM80_Device_GemmPlanarComplex_f16n_f16t_f16n_tensor_op_f32_16816, 64x64x32_32x32x32) {
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_f16_s16816_nt_base>;
+
+  EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
 using gemm_planar_complex_s16816_ch_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
@@ -213,4 +342,46 @@ TEST(SM80_Device_GemmPlanarComplex_f16c_f16h_f32n_tensor_op_f32_16816, 64x64x32_
 
 ////////////////////////////////////////////////////////////////////////////////
 
+using gemm_planar_complex_cf16_s16816_ch_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
+  cutlass::half_t,
+  cutlass::layout::ColumnMajor,
+  cutlass::ComplexTransform::kConjugate,
+  8,
+  cutlass::half_t,
+  cutlass::layout::RowMajor,
+  cutlass::ComplexTransform::kConjugate,
+  8,
+  cutlass::half_t,
+  cutlass::layout::RowMajor,
+  float,
+  cutlass::arch::OpClassTensorOp,
+  cutlass::arch::Sm80,
+  cutlass::gemm::GemmShape<64, 64, 32>,
+  cutlass::gemm::GemmShape<32, 32, 32>,
+  cutlass::gemm::GemmShape<16, 8, 16>,
+  cutlass::epilogue::thread::LinearCombinationPlanarComplex<
+    float,
+    4,
+    float,
+    float
+  >,
+  cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+  3,
+  cutlass::arch::OpMultiplyAdd
+>::GemmKernel;
+
+struct gemm_planar_complex_cf16_s16816_ch : gemm_planar_complex_cf16_s16816_ch_base {
+
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM80_Device_GemmPlanarComplex_f16c_f16h_f16n_tensor_op_f32_16816, 64x64x32_32x32x32) {
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_cf16_s16816_ch>;
+
+  EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
+}
+////////////////////////////////////////////////////////////////////////////////
+
 #endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)

test/unit/gemm/device/gemm_with_broadcast_f16n_f16n_f16n_tensorop_f32_sm75.cu

@@ -0,0 +1,458 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+/*! \file
+    \brief Tests for device-wide GEMM interface
+*/
+
+#include <iostream>
+
+#include "cutlass/cutlass.h"
+#include "cutlass/functional.h"
+
+#include "cutlass/gemm/kernel/default_gemm_with_broadcast.h"
+#include "cutlass/gemm/device/gemm_universal_adapter.h"
+
+#include "cutlass/epilogue/thread/linear_combination_bias_elementwise.h"
+#include "cutlass/epilogue/thread/linear_combination_bias_relu.h"
+
+#include "../../common/cutlass_unit_test.h"
+
+#include "cutlass/util/host_tensor.h"
+#include "cutlass/util/tensor_view_io.h"
+#include "cutlass/util/reference/host/tensor_fill.h"
+#include "cutlass/util/reference/host/tensor_copy.h"
+#include "cutlass/util/reference/host/tensor_compare.h"
+#include "cutlass/util/reference/host/gemm.h"
+
+#include "testbed_gemm_with_broadcast.h"
+
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+/// Computes:
+///
+///  Z = GEMM+Bias+ReLu
+///  T = Relu conditional
+///
+template <typename Gemm>
+struct GemmWithBiasReluReferenceOp {
+
+  using OutputOp = typename Gemm::GemmKernel::Epilogue::OutputOp;
+
+  using ElementCompute = typename OutputOp::ElementCompute;
+  using ElementZ = typename OutputOp::ElementZ;
+  using ElementT = typename OutputOp::ElementT;
+
+  typename OutputOp::BinaryOp binary_op;
+  typename OutputOp::ElementwiseOp elementwise_op;
+
+  GemmWithBiasReluReferenceOp() { }
+
+  void operator()(ElementZ &Z, ElementT &T, ElementCompute gemm, ElementCompute bias) {
+    
+    ElementCompute kThreshold = ElementCompute();
+
+    ElementCompute z_full = binary_op(gemm, bias);
+    
+    bool conditional = (z_full >= kThreshold);
+
+    if (!conditional) {
+      z_full = kThreshold;
+    }
+
+    Z = ElementZ(z_full);
+    T = ElementT(conditional);
+  }
+};
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+#if defined(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM75_Device_GemmWithBroadcast_GELU_f16n_f16n_f16n_tensor_op_f32, 128x128x32_64x64x8) {
+  
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasElementwise<
+    cutlass::half_t,
+    float,
+    float,
+    cutlass::half_t,
+    cutlass::half_t,
+    8,
+    cutlass::epilogue::thread::GELU_taylor<float>
+  >;
+
+  using GemmKernel = 
+    typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed B operand
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed A operand
+      cutlass::half_t, cutlass::layout::RowMajor,
+      float,
+      cutlass::arch::OpClassTensorOp,
+      cutlass::arch::Sm75,
+      cutlass::gemm::GemmShape<128, 128, 32>,
+      cutlass::gemm::GemmShape<64, 64, 32>,
+      cutlass::gemm::GemmShape<16, 8, 8>,
+      EpilogueOutputOp,
+      cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
+      2,
+      cutlass::arch::OpMultiplyAdd
+  >::GemmKernel;
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+
+  test::gemm::device::TestAllGemmWithBroadcast<Gemm>();
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM70_Device_GemmWithBroadcast_GELU_f16n_f16n_f16n_tensor_op_f32, 128x128x32_64x64x8) {
+  
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasElementwise<
+    cutlass::half_t,
+    float,
+    float,
+    cutlass::half_t,
+    cutlass::half_t,
+    8,
+    cutlass::epilogue::thread::GELU_taylor<float>
+  >;
+
+  using GemmKernel = 
+    typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed B operand
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed A operand
+      cutlass::half_t, cutlass::layout::RowMajor,
+      float,
+      cutlass::arch::OpClassTensorOp,
+      cutlass::arch::Sm70,
+      cutlass::gemm::GemmShape<128, 128, 32>,
+      cutlass::gemm::GemmShape<64, 64, 32>,
+      cutlass::gemm::GemmShape<8, 8, 4>,
+      EpilogueOutputOp,
+      cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
+      2,
+      cutlass::arch::OpMultiplyAdd
+  >::GemmKernel;
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+
+  test::gemm::device::TestAllGemmWithBroadcast<Gemm>();
+}
+
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM75_Device_GemmWithBroadcast_RELU_f16n_f16n_f16n_tensor_op_f32, 128x128x32_64x64x8) {
+  
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasRelu<
+    cutlass::half_t,
+    float,
+    float,
+    cutlass::half_t,
+    8,
+    true
+  >;
+
+  using GemmKernel = 
+    typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed B operand
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed A operand
+      cutlass::half_t, cutlass::layout::RowMajor,
+      float,
+      cutlass::arch::OpClassTensorOp,
+      cutlass::arch::Sm75,
+      cutlass::gemm::GemmShape<128, 128, 32>,
+      cutlass::gemm::GemmShape<64, 64, 32>,
+      cutlass::gemm::GemmShape<16, 8, 8>,
+      EpilogueOutputOp,
+      cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
+      2,
+      cutlass::arch::OpMultiplyAdd
+  >::GemmKernel;
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+
+  test::gemm::device::TestAllGemmWithBroadcast<Gemm, GemmWithBiasReluReferenceOp<Gemm> >();
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM70_Device_GemmWithBroadcast_RELU_f16n_f16n_f16n_tensor_op_f32, 128x128x32_64x64x8) {
+  
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasRelu<
+    cutlass::half_t,
+    float,
+    float,
+    cutlass::half_t,
+    8,
+    true
+  >;
+
+  using GemmKernel = 
+    typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed B operand
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed A operand
+      cutlass::half_t, cutlass::layout::RowMajor,
+      float,
+      cutlass::arch::OpClassTensorOp,
+      cutlass::arch::Sm70,
+      cutlass::gemm::GemmShape<128, 128, 32>,
+      cutlass::gemm::GemmShape<64, 64, 32>,
+      cutlass::gemm::GemmShape<8, 8, 4>,
+      EpilogueOutputOp,
+      cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
+      2,
+      cutlass::arch::OpMultiplyAdd
+  >::GemmKernel;
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+
+  test::gemm::device::TestAllGemmWithBroadcast<Gemm, GemmWithBiasReluReferenceOp<Gemm> >();
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+#endif  // if defiend(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM80_Device_GemmWithBroadcast_GELU_f16n_f16n_f16n_tensor_op_f32, 128x128_32x5_64x64x32_16x8x16) {
+  
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasElementwise<
+    cutlass::half_t,
+    float,
+    float,
+    cutlass::half_t,
+    cutlass::half_t,
+    8,
+    cutlass::epilogue::thread::GELU_taylor<float>
+  >;
+
+  using GemmKernel = 
+    typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed B operand
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed A operand
+      cutlass::half_t, cutlass::layout::RowMajor,
+      float,
+      cutlass::arch::OpClassTensorOp,
+      cutlass::arch::Sm80,
+      cutlass::gemm::GemmShape<128, 128, 32>,
+      cutlass::gemm::GemmShape<64, 64, 32>,
+      cutlass::gemm::GemmShape<16, 8, 16>,
+      EpilogueOutputOp,
+      cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
+      5,
+      cutlass::arch::OpMultiplyAdd
+  >::GemmKernel;
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+
+  test::gemm::device::TestAllGemmWithBroadcast<Gemm>();
+}
+
+TEST(SM80_Device_GemmWithBroadcast_RELU_f16n_f16n_f16n_tensor_op_f32, 128x128_32x5_64x64x32_16x8x16) {
+  
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasRelu<
+    cutlass::half_t,
+    float,
+    float,
+    cutlass::half_t,
+    8,
+    true
+  >;
+
+  using GemmKernel = 
+    typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed B operand
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed A operand
+      cutlass::half_t, cutlass::layout::RowMajor,
+      float,
+      cutlass::arch::OpClassTensorOp,
+      cutlass::arch::Sm80,
+      cutlass::gemm::GemmShape<128, 128, 32>,
+      cutlass::gemm::GemmShape<64, 64, 32>,
+      cutlass::gemm::GemmShape<16, 8, 16>,
+      EpilogueOutputOp,
+      cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
+      5,
+      cutlass::arch::OpMultiplyAdd
+  >::GemmKernel;
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+
+  test::gemm::device::TestAllGemmWithBroadcast<Gemm, GemmWithBiasReluReferenceOp<Gemm>>();
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM80_Device_GemmWithBroadcast_GELU_f16n_f16n_f16n_tensor_op_f32, 128x128_32x4_64x64x32_16x8x16) {
+  
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasElementwise<
+    cutlass::half_t,
+    float,
+    float,
+    cutlass::half_t,
+    cutlass::half_t,
+    8,
+    cutlass::epilogue::thread::GELU_taylor<float>
+  >;
+
+  using GemmKernel = 
+    typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed B operand
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed A operand
+      cutlass::half_t, cutlass::layout::RowMajor,
+      float,
+      cutlass::arch::OpClassTensorOp,
+      cutlass::arch::Sm80,
+      cutlass::gemm::GemmShape<128, 128, 32>,
+      cutlass::gemm::GemmShape<64, 64, 32>,
+      cutlass::gemm::GemmShape<16, 8, 16>,
+      EpilogueOutputOp,
+      cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
+      4,
+      cutlass::arch::OpMultiplyAdd
+  >::GemmKernel;
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+
+  test::gemm::device::TestAllGemmWithBroadcast<Gemm>();
+}
+
+TEST(SM80_Device_GemmWithBroadcast_RELU_f16n_f16n_f16n_tensor_op_f32, 128x128_32x4_64x64x32_16x8x16) {
+  
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasRelu<
+    cutlass::half_t,
+    float,
+    float,
+    cutlass::half_t,
+    8,
+    true
+  >;
+
+  using GemmKernel = 
+    typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed B operand
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed A operand
+      cutlass::half_t, cutlass::layout::RowMajor,
+      float,
+      cutlass::arch::OpClassTensorOp,
+      cutlass::arch::Sm80,
+      cutlass::gemm::GemmShape<128, 128, 32>,
+      cutlass::gemm::GemmShape<64, 64, 32>,
+      cutlass::gemm::GemmShape<16, 8, 16>,
+      EpilogueOutputOp,
+      cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
+      4,
+      cutlass::arch::OpMultiplyAdd
+  >::GemmKernel;
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+
+  test::gemm::device::TestAllGemmWithBroadcast<Gemm, GemmWithBiasReluReferenceOp<Gemm>>();
+}
+
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM80_Device_GemmWithBroadcast_GELU_f16n_f16n_f16n_tensor_op_f32, 128x128_32x3_64x64x32_16x8x16) {
+  
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasElementwise<
+    cutlass::half_t,
+    float,
+    float,
+    cutlass::half_t,
+    cutlass::half_t,
+    8,
+    cutlass::epilogue::thread::GELU_taylor<float>
+  >;
+
+  using GemmKernel = 
+    typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed B operand
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed A operand
+      cutlass::half_t, cutlass::layout::RowMajor,
+      float,
+      cutlass::arch::OpClassTensorOp,
+      cutlass::arch::Sm80,
+      cutlass::gemm::GemmShape<128, 128, 32>,
+      cutlass::gemm::GemmShape<64, 64, 32>,
+      cutlass::gemm::GemmShape<16, 8, 16>,
+      EpilogueOutputOp,
+      cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
+      3,
+      cutlass::arch::OpMultiplyAdd
+  >::GemmKernel;
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+
+  test::gemm::device::TestAllGemmWithBroadcast<Gemm>();
+}
+
+TEST(SM80_Device_GemmWithBroadcast_RELU_f16n_f16n_f16n_tensor_op_f32, 128x128_32x3_64x64x32_16x8x16) {
+  
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasRelu<
+    cutlass::half_t,
+    float,
+    float,
+    cutlass::half_t,
+    8,
+    true
+  >;
+
+  using GemmKernel = 
+    typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed B operand
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed A operand
+      cutlass::half_t, cutlass::layout::RowMajor,
+      float,
+      cutlass::arch::OpClassTensorOp,
+      cutlass::arch::Sm80,
+      cutlass::gemm::GemmShape<128, 128, 32>,
+      cutlass::gemm::GemmShape<64, 64, 32>,
+      cutlass::gemm::GemmShape<16, 8, 16>,
+      EpilogueOutputOp,
+      cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
+      3,
+      cutlass::arch::OpMultiplyAdd
+  >::GemmKernel;
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+
+  test::gemm::device::TestAllGemmWithBroadcast<Gemm, GemmWithBiasReluReferenceOp<Gemm> >();
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+#endif
+
+/////////////////////////////////////////////////////////////////////////////////////////////////

test/unit/gemm/device/gemm_with_reduction_f16n_f16n_f16n_tensorop_f32_sm75.cu

@@ -0,0 +1,378 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+/*! \file
+    \brief Tests for device-wide GEMM interface
+*/
+
+#include <iostream>
+
+#include "cutlass/cutlass.h"
+#include "cutlass/functional.h"
+
+#include "cutlass/gemm/kernel/default_gemm_with_reduction.h"
+#include "cutlass/gemm/device/gemm_universal_adapter.h"
+
+#include "cutlass/epilogue/thread/linear_combination_drelu.h"
+#include "cutlass/epilogue/thread/linear_combination_dgelu.h"
+
+#include "../../common/cutlass_unit_test.h"
+
+#include "cutlass/util/host_tensor.h"
+#include "cutlass/util/tensor_view_io.h"
+#include "cutlass/util/reference/host/tensor_fill.h"
+#include "cutlass/util/reference/host/tensor_copy.h"
+#include "cutlass/util/reference/host/tensor_compare.h"
+#include "cutlass/util/reference/host/gemm.h"
+
+#include "testbed_gemm_with_reduction.h"
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+#if defined(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+struct dReluLambda {
+  float operator()(float d_y, float t) {
+    if (t <= 0) {
+      d_y = 0;
+    }
+    return d_y;
+  }
+};
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM75_Device_GemmWithReduction_dReLU_bGrad_f16n_f16n_f16n_tensor_op_f32, 128x128x32_64x64x8) {
+  
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
+    float,
+    float,
+    cutlass::half_t,
+    cutlass::half_t,
+    8
+  >;
+
+  using GemmKernel = 
+    typename cutlass::gemm::kernel::DefaultGemmWithReduction<
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed B operand
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed A operand
+      cutlass::half_t, cutlass::layout::RowMajor,
+      float,
+      cutlass::arch::OpClassTensorOp,
+      cutlass::arch::Sm75,
+      cutlass::gemm::GemmShape<128, 128, 32>,
+      cutlass::gemm::GemmShape<64, 64, 32>,
+      cutlass::gemm::GemmShape<16, 8, 8>,
+      EpilogueOutputOp,
+      cutlass::plus<float>,
+      cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
+      2,
+      cutlass::arch::OpMultiplyAdd
+  >::GemmKernel;
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+
+  using ReferenceOp = test::gemm::device::GemmWithReductionReference<
+    Gemm, 
+    dReluLambda
+  >;
+
+  test::gemm::device::TestGemmWithReduction<Gemm, ReferenceOp>(
+    {520, 264, 96},
+    cutlass::gemm::GemmUniversalMode::kGemm,
+    2,
+    float(1.25),
+    float(2.25)
+  );
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM75_Device_GemmWithReduction_dReLU_bGrad_f16n_f16n_f16n_tensor_op_f32, 256x128x32_64x64x8) {
+  
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
+    float,
+    float,
+    cutlass::half_t,
+    cutlass::half_t,
+    8
+  >;
+
+  using GemmKernel = 
+    typename cutlass::gemm::kernel::DefaultGemmWithReduction<
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed B operand
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed A operand
+      cutlass::half_t, cutlass::layout::RowMajor,
+      float,
+      cutlass::arch::OpClassTensorOp,
+      cutlass::arch::Sm75,
+      cutlass::gemm::GemmShape<256, 128, 32>,
+      cutlass::gemm::GemmShape<64, 64, 32>,
+      cutlass::gemm::GemmShape<16, 8, 8>,
+      EpilogueOutputOp,
+      cutlass::plus<float>,
+      cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
+      2,
+      cutlass::arch::OpMultiplyAdd
+  >::GemmKernel;
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+
+  using ReferenceOp = test::gemm::device::GemmWithReductionReference<
+    Gemm, 
+    dReluLambda
+  >;
+
+  test::gemm::device::TestGemmWithReduction<Gemm, ReferenceOp>(
+    {520, 264, 96},
+    cutlass::gemm::GemmUniversalMode::kGemm,
+    1,
+    float(1.25),
+    float(2.25)
+  );
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM70_Device_GemmWithReduction_dReLU_bGrad_f16n_f16n_f16n_tensor_op_f32, 128x128x32_64x64x8) {
+  
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
+    float,
+    float,
+    cutlass::half_t,
+    cutlass::half_t,
+    8
+  >;
+
+  using GemmKernel = 
+    typename cutlass::gemm::kernel::DefaultGemmWithReduction<
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed B operand
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed A operand
+      cutlass::half_t, cutlass::layout::RowMajor,
+      float,
+      cutlass::arch::OpClassTensorOp,
+      cutlass::arch::Sm70,
+      cutlass::gemm::GemmShape<128, 128, 32>,
+      cutlass::gemm::GemmShape<64, 64, 32>,
+      cutlass::gemm::GemmShape<8, 8, 4>,
+      EpilogueOutputOp,
+      cutlass::plus<float>,
+      cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
+      2,
+      cutlass::arch::OpMultiplyAdd
+  >::GemmKernel;
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+
+  using ReferenceOp = test::gemm::device::GemmWithReductionReference<
+    Gemm, 
+    dReluLambda
+  >;
+  
+  test::gemm::device::TestGemmWithReduction<Gemm, ReferenceOp>(
+    {520, 264, 96},
+    cutlass::gemm::GemmUniversalMode::kGemm,
+    2,
+    float(1.25),
+    float(2.25)
+  );
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM70_Device_GemmWithReduction_dReLU_bGrad_f16n_f16n_f16n_tensor_op_f32, 256x128x32_64x64x8) {
+  
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
+    float,
+    float,
+    cutlass::half_t,
+    cutlass::half_t,
+    8
+  >;
+
+  using GemmKernel = 
+    typename cutlass::gemm::kernel::DefaultGemmWithReduction<
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed B operand
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed A operand
+      cutlass::half_t, cutlass::layout::RowMajor,
+      float,
+      cutlass::arch::OpClassTensorOp,
+      cutlass::arch::Sm70,
+      cutlass::gemm::GemmShape<256, 128, 32>,
+      cutlass::gemm::GemmShape<64, 64, 32>,
+      cutlass::gemm::GemmShape<8, 8, 4>,
+      EpilogueOutputOp,
+      cutlass::plus<float>,
+      cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
+      2,
+      cutlass::arch::OpMultiplyAdd
+  >::GemmKernel;
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+
+  using ReferenceOp = test::gemm::device::GemmWithReductionReference<
+    Gemm, 
+    dReluLambda
+  >;
+  
+  test::gemm::device::TestGemmWithReduction<Gemm, ReferenceOp>(
+    {520, 264, 96},
+    cutlass::gemm::GemmUniversalMode::kGemm,
+    1,
+    float(1.25),
+    float(2.25)
+  );
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace test {
+namespace gemm {
+namespace device {
+
+template <typename Gemm>
+struct Gemm_dReLU_packed_bits_reference_op {
+  using ElementAccumulator = typename Gemm::ElementAccumulator;
+  using ElementCompute = typename Gemm::GemmKernel::Epilogue::ElementCompute;
+  using ElementC = typename Gemm::ElementC;
+  using ElementT = typename Gemm::GemmKernel::Epilogue::ElementTensor;
+
+  //
+  // Methods
+  //
+
+  Gemm_dReLU_packed_bits_reference_op() { }
+
+  ElementCompute operator()(
+    ElementAccumulator d_y, 
+    ElementT t) const {
+    
+    ElementCompute result = ElementCompute(d_y);
+
+    bool cond = bool(t);
+    if (!cond) {
+      result = ElementCompute();
+    }
+
+    return result;
+  }
+};
+
+} // namespace device
+} // namespace gemm
+} // namespace test
+
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM75_Device_GemmWithReduction_dReLU_conditional_bits_bGrad_f16n_f16n_f16n_tensor_op_f32, 128x128x32_64x64x8) {
+  
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationDReluConditionalBits<
+    float,
+    float,
+    cutlass::half_t,
+    8
+  >;
+
+  using GemmKernel = 
+    typename cutlass::gemm::kernel::DefaultGemmWithReduction<
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed B operand
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed A operand
+      cutlass::half_t, cutlass::layout::RowMajor,
+      float,
+      cutlass::arch::OpClassTensorOp,
+      cutlass::arch::Sm75,
+      cutlass::gemm::GemmShape<128, 128, 32>,
+      cutlass::gemm::GemmShape<64, 64, 32>,
+      cutlass::gemm::GemmShape<16, 8, 8>,
+      EpilogueOutputOp,
+      cutlass::plus<float>,
+      cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
+      2,
+      cutlass::arch::OpMultiplyAdd
+  >::GemmKernel;
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+
+  using ReferenceOp = test::gemm::device::Gemm_dReLU_packed_bits_reference_op<Gemm>;
+
+  test::gemm::device::TestGemmWithReduction<Gemm, ReferenceOp>(
+    {520, 264, 96},
+    cutlass::gemm::GemmUniversalMode::kGemm,
+    2,
+    float(1.25),
+    float(2.25)
+  );
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM70_Device_GemmWithReduction_dReLU_conditional_bits_bGrad_f16n_f16n_f16n_tensor_op_f32, 128x128x32_64x64x8) {
+  
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationDReluConditionalBits<
+    float,
+    float,
+    cutlass::half_t,
+    8
+  >;
+
+  using GemmKernel = 
+    typename cutlass::gemm::kernel::DefaultGemmWithReduction<
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed B operand
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed A operand
+      cutlass::half_t, cutlass::layout::RowMajor,
+      float,
+      cutlass::arch::OpClassTensorOp,
+      cutlass::arch::Sm70,
+      cutlass::gemm::GemmShape<128, 128, 32>,
+      cutlass::gemm::GemmShape<64, 64, 32>,
+      cutlass::gemm::GemmShape<8, 8, 4>,
+      EpilogueOutputOp,
+      cutlass::plus<float>,
+      cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
+      2,
+      cutlass::arch::OpMultiplyAdd
+  >::GemmKernel;
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+
+  using ReferenceOp = test::gemm::device::Gemm_dReLU_packed_bits_reference_op<Gemm>;
+
+  test::gemm::device::TestGemmWithReduction<Gemm, ReferenceOp>(
+    {520, 264, 96},
+    cutlass::gemm::GemmUniversalMode::kGemm,
+    2,
+    float(1.25),
+    float(2.25)
+  );
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+#endif  // if defiend(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
+
+/////////////////////////////////////////////////////////////////////////////////////////////////

test/unit/gemm/device/gemm_with_reduction_f16t_f16n_f16n_tensorop_f32_sm80.cu

@@ -0,0 +1,112 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+/*! \file
+    \brief Tests for device-wide GEMM interface
+*/
+
+#include <iostream>
+
+#include "cutlass/cutlass.h"
+#include "cutlass/functional.h"
+
+#include "cutlass/gemm/kernel/default_gemm_with_reduction.h"
+#include "cutlass/gemm/device/gemm_universal_adapter.h"
+
+#include "cutlass/epilogue/thread/linear_combination_drelu.h"
+#include "cutlass/epilogue/thread/linear_combination_dgelu.h"
+
+#include "../../common/cutlass_unit_test.h"
+
+#include "cutlass/util/host_tensor.h"
+#include "cutlass/util/tensor_view_io.h"
+#include "cutlass/util/reference/host/tensor_fill.h"
+#include "cutlass/util/reference/host/tensor_copy.h"
+#include "cutlass/util/reference/host/tensor_compare.h"
+#include "cutlass/util/reference/host/gemm.h"
+
+#include "testbed_gemm_with_reduction.h"
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+struct dReluLambda {
+  float operator()(float d_y, float t) {
+    if (t <= 0) {
+      d_y = 0;
+    }
+    return d_y;
+  }
+};
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM80_Device_GemmWithReduction_dReLU_bGrad_f16t_f16n_f16n_tensor_op_f32, 128x128x32_64x64x32) {
+  
+  using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
+    float,
+    float,
+    cutlass::half_t,
+    cutlass::half_t,
+    8
+  >;
+
+  using GemmKernel = 
+    typename cutlass::gemm::kernel::DefaultGemmWithReduction<
+      cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8,    // transposed B operand
+      cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::ComplexTransform::kNone, 8,    // transposed A operand
+      cutlass::half_t, cutlass::layout::RowMajor,
+      float,
+      cutlass::arch::OpClassTensorOp,
+      cutlass::arch::Sm80,
+      cutlass::gemm::GemmShape<128, 128, 32>,
+      cutlass::gemm::GemmShape<64, 64, 32>,
+      cutlass::gemm::GemmShape<16, 8, 16>,
+      EpilogueOutputOp,
+      cutlass::plus<float>,
+      cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
+      5,
+      cutlass::arch::OpMultiplyAdd
+  >::GemmKernel;
+
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+
+  using ReferenceOp = test::gemm::device::GemmWithReductionReference<
+    Gemm, 
+    dReluLambda
+  >;
+
+  test::gemm::device::TestGemmWithReduction<Gemm, ReferenceOp>(
+    {8, 8, 136},
+    cutlass::gemm::GemmUniversalMode::kGemm
+  );
+}
+
+#endif  // if defiend(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
+
+/////////////////////////////////////////////////////////////////////////////////////////////////

test/unit/gemm/device/gemv.cu

@@ -0,0 +1,438 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+/*! \file
+    \brief Tests for device-wide GEMV interface
+*/
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+
+#include "cutlass/cutlass.h"
+#include "cutlass/gemm/kernel/gemv.h"
+#include "cutlass/gemm/device/gemv.h"
+
+#include "../../common/cutlass_unit_test.h"
+
+#include "cutlass/util/host_tensor.h"
+#include "cutlass/util/tensor_view_io.h"
+#include "cutlass/util/distribution.h"
+#include "cutlass/util/reference/host/tensor_fill.h"
+#include "cutlass/util/reference/host/tensor_copy.h"
+#include "cutlass/util/reference/host/tensor_compare.h"
+#include "cutlass/util/reference/host/tensor_norm.h"
+#include "cutlass/util/reference/host/gemm.h"
+#include "cutlass/util/reference/host/gemm_complex.h"
+
+#include "testbed_utils.h"
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace test {
+namespace gemm {
+
+template <typename Gemv>
+class TestbedGemv {
+public:
+
+  using ElementA = typename Gemv::ElementA;
+  using LayoutA  = typename Gemv::LayoutA;
+  using ElementB = typename Gemv::ElementB;
+  using ElementC = typename Gemv::ElementC;
+
+  using ElementAccumulator = typename Gemv::ElementAccumulator;
+  using ElementCompute = typename Gemv::EpilogueOutputOp::ElementCompute;
+
+  using LayoutV = cutlass::layout::RowMajor;
+
+private:
+
+  /// Initialization
+  cutlass::Distribution::Kind init_A;
+  cutlass::Distribution::Kind init_B;
+  cutlass::Distribution::Kind init_C;
+  uint64_t seed;
+
+  cutlass::HostTensor<ElementA, LayoutA> tensor_A;
+  cutlass::HostTensor<ElementB, LayoutV> tensor_B;
+  cutlass::HostTensor<ElementC, LayoutV> tensor_C;
+  cutlass::HostTensor<ElementC, LayoutV> tensor_D;
+  cutlass::HostTensor<ElementC, LayoutV> reference_D;
+
+public:
+
+  //
+  // Methods
+  //
+
+  TestbedGemv(
+    cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
+    cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
+    cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
+    uint64_t seed_ = 2080
+  ):
+    init_A(init_A_), init_B(init_B_), init_C(init_C_), seed(seed_) { }
+
+  /// Helper to initialize a tensor view
+  template <typename Element, typename Layout>
+  bool initialize_tensor(
+    cutlass::TensorView<Element, Layout> view, 
+    cutlass::Distribution::Kind dist_kind,
+    uint64_t seed) {
+
+    if (dist_kind == cutlass::Distribution::Uniform) {
+
+      double scope_max, scope_min;
+      int bits_input = cutlass::sizeof_bits<Element>::value;
+      int bits_output = cutlass::sizeof_bits<typename Gemv::ElementC>::value;
+
+      if (bits_input == 1) {
+        scope_max = 2;
+        scope_min = 0;
+      } else if (bits_input <= 8) {
+        scope_max = 2;
+        scope_min = -2;
+      } else if (bits_output == 16) {
+        scope_max = 5;
+        scope_min = -5;
+      } else {
+        scope_max = 8;
+        scope_min = -8;
+      }
+
+      cutlass::reference::host::TensorFillRandomUniform(
+        view, seed, scope_max, scope_min, 0);
+    } 
+    else if (dist_kind == cutlass::Distribution::Identity) {
+
+      cutlass::reference::host::TensorFillIdentity(view);
+    } 
+    else if (dist_kind == cutlass::Distribution::Gaussian) {
+
+      cutlass::reference::host::TensorFillRandomGaussian(view, seed, 0, 0.5);
+    }
+    else if (dist_kind == cutlass::Distribution::Sequential) {
+
+      cutlass::reference::host::BlockFillSequential(
+        view.data(), view.capacity());
+    } 
+    else {
+      // TODO: Implement the rest
+      EXPECT_TRUE(false) << "Not implemented";
+      return false;
+    }
+
+    return true;
+  }
+
+  /// Initializes data structures
+  void initialize(
+    cutlass::MatrixCoord problem_size
+  ) {
+
+    //
+    // Allocate the GEMM workspace
+    //
+
+    tensor_A.resize(problem_size);
+    tensor_B.resize({problem_size.column(), 1});
+    tensor_C.resize({problem_size.row(), 1});
+    tensor_D.resize({problem_size.row(), 1});
+    reference_D.resize({problem_size.row(), 1}, false);
+
+    EXPECT_TRUE(initialize_tensor(tensor_A.host_view(), init_A, seed + 2019));
+    EXPECT_TRUE(initialize_tensor(tensor_B.host_view(), init_B, seed + 2018));
+    EXPECT_TRUE(initialize_tensor(tensor_C.host_view(), init_C, seed + 2017));
+
+    // It is possible to randomly initialize to all zeros, so override this with non-zeros
+    // in the upper left corner of each operand.
+    tensor_A.host_view().at({0, 0}) = typename Gemv::ElementA(1);
+    tensor_B.host_view().at({0, 0}) = typename Gemv::ElementB(1);
+    tensor_C.host_view().at({0, 0}) = typename Gemv::ElementC(1);
+
+    cutlass::reference::host::TensorCopy(reference_D.host_view(), tensor_C.host_view());
+
+    tensor_A.sync_device();
+    tensor_B.sync_device();
+    tensor_C.sync_device();
+    tensor_D.sync_device();
+  }  
+
+  /// Compares computed reference with device reference and outputs to a file if incorrect
+  bool compare_reference(
+    cutlass::MatrixCoord problem_size, 
+    ElementCompute alpha, 
+    ElementCompute beta) {
+
+    tensor_D.sync_host();
+
+    EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_A.host_view()), 0);
+    EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_B.host_view()), 0);
+    EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_C.host_view()), 0);
+    
+    EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_D.host_view()), 0);
+    EXPECT_GT(cutlass::reference::host::TensorNorm(reference_D.host_view()), 0);
+
+    bool passed = cutlass::reference::host::TensorEquals(reference_D.host_view(), tensor_D.host_view());
+
+    EXPECT_TRUE(passed) << " mismatched reference";
+
+    if (!passed) {
+
+      std::ofstream file("testbed_universal_errors.txt");
+
+      file
+        << "problem: " << problem_size 
+        << ", alpha: " << alpha << ", beta: " << beta << "\n\n";
+
+      file 
+        << "A =\n" << tensor_A.host_view()
+        << "\nB =\n" << tensor_B.host_view()
+        << "\nC =\n" << tensor_C.host_view()
+        << "\n\nReference =\n" << reference_D.host_view()
+        << "\nComputed =\n" << tensor_D.host_view();
+    }
+
+    return passed;
+  }
+
+  /// Verifies the result is a GEMM
+  bool verify(
+    cutlass::MatrixCoord problem_size, 
+    ElementCompute alpha, 
+    ElementCompute beta) {
+
+    //
+    // Verify
+    //
+
+    cutlass::reference::host::GemmComplex<
+        typename Gemv::ElementA, typename Gemv::LayoutA,
+        typename Gemv::ElementB, LayoutV,
+        typename Gemv::ElementC, LayoutV, 
+        ElementCompute, ElementAccumulator
+    >(
+      {problem_size.row(), 1, problem_size.column()},
+      alpha, 
+      tensor_A.host_ref(),
+      Gemv::kTransformA,
+      tensor_B.host_ref(),
+      Gemv::kTransformB,
+      beta, 
+      tensor_C.host_ref(), 
+      reference_D.host_ref(), 
+      ElementAccumulator(0)
+    );
+
+    return compare_reference(problem_size, alpha, beta);
+  }
+
+  /// Runs one problem size
+  bool run(
+    cutlass::MatrixCoord problem_size, 
+    ElementCompute alpha,
+    ElementCompute beta) {
+
+    this->initialize(problem_size);
+
+    //
+    // Initialize the GEMM operator
+    //
+
+    typename Gemv::Arguments arguments{
+      problem_size,
+      {alpha, beta},
+      tensor_A.device_ref(),
+      tensor_B.device_data(),
+      tensor_C.device_data(),
+      tensor_D.device_data(),
+      tensor_B.layout().stride(0),
+      tensor_C.layout().stride(0),
+      tensor_D.layout().stride(0)
+    };
+
+    Gemv gemm_op;
+
+    size_t workspace_size = Gemv::get_workspace_size(arguments);
+
+    cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
+
+    cutlass::Status status = gemm_op.initialize(arguments, workspace.get());
+
+    EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
+
+    //
+    // Run the GEMM
+    //
+
+    status = gemm_op();
+
+    EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
+
+    //
+    // Verify
+    //
+
+    bool passed = this->verify(problem_size, alpha, beta);
+
+    return passed;
+  }
+};
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename Gemv> 
+bool TestAllGemv() {
+
+  using ElementCompute = typename Gemv::EpilogueOutputOp::ElementCompute;
+
+  int M[] = {
+    8, 48, 192, 520
+  };
+
+  int K[] = {
+    8, 192, 528
+  };
+
+  double Alpha[] = {
+    1, 1.25
+  };
+
+  double Beta[] = {
+    0, 1, 1.25
+  };
+
+  for (int m : M) {
+    for (int k : K) {
+      for (double alpha : Alpha) {
+        for (double beta : Beta) {
+
+          TestbedGemv<Gemv> testbed;
+
+          if (!testbed.run({m, k}, ElementCompute(alpha), ElementCompute(beta))) {
+            return false;
+          }
+        }
+      }
+    }
+  }
+
+  return true;
+}
+
+} // namespace gemm
+} // namespace test
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM50_Device_Gemv_f32n_f32_f32_simt_f32, Simple) {
+
+  using ElementOutput = float;
+  using LayoutA = cutlass::layout::ColumnMajor;
+  using ElementAccumulator = float;
+
+  using EpilogueOp = cutlass::epilogue::thread::LinearCombination<
+      ElementOutput,
+      1,
+      ElementAccumulator, 
+      ElementAccumulator>;
+
+  using Gemv = cutlass::gemm::device::Gemv<
+    cutlass::gemm::kernel::Gemv<
+        ElementOutput,          // Element A
+        LayoutA,                // Layout A
+        ElementOutput,          // Element B
+        ElementOutput,          // Element C
+        ElementAccumulator,     // Element Accumulator
+        EpilogueOp              // Output operator
+        >
+    >;
+
+
+  EXPECT_TRUE(test::gemm::TestAllGemv<Gemv>());
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM50_Device_Gemv_f16n_f16_f32_simt_f32, Simple) {
+
+  using ElementInput = cutlass::half_t;
+  using ElementOutput = float;
+  using LayoutA = cutlass::layout::ColumnMajor;
+  using ElementAccumulator = float;
+
+  using EpilogueOp = cutlass::epilogue::thread::LinearCombination<
+      ElementOutput,
+      1,
+      ElementAccumulator, 
+      ElementAccumulator>;
+
+  using Gemv = cutlass::gemm::device::Gemv<
+    cutlass::gemm::kernel::Gemv<
+        ElementInput,           // Element A
+        LayoutA,                // Layout A
+        ElementInput,           // Element B
+        ElementOutput,          // Element C
+        ElementAccumulator,     // Element Accumulator
+        EpilogueOp              // Output operator
+        >
+    >;
+
+
+  EXPECT_TRUE(test::gemm::TestAllGemv<Gemv>());
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM50_Device_Gemv_f16n_f16_f16_simt_f32, Simple) {
+
+  using ElementInput = cutlass::half_t;
+  using ElementOutput = cutlass::half_t;
+  using LayoutA = cutlass::layout::ColumnMajor;
+  using ElementAccumulator = float;
+
+  using EpilogueOp = cutlass::epilogue::thread::LinearCombination<
+      ElementOutput,
+      1,
+      ElementAccumulator, 
+      ElementAccumulator>;
+
+  using Gemv = cutlass::gemm::device::Gemv<
+    cutlass::gemm::kernel::Gemv<
+        ElementInput,           // Element A
+        LayoutA,                // Layout A
+        ElementInput,           // Element B
+        ElementOutput,          // Element C
+        ElementAccumulator,     // Element Accumulator
+        EpilogueOp              // Output operator
+        >
+    >;
+
+
+  EXPECT_TRUE(test::gemm::TestAllGemv<Gemv>());
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////

test/unit/gemm/device/simt_cgemm_nn_sm50.cu

@@ -673,66 +673,6 @@ CUTLASS_TEST_L1(SM50_device_cgemm_nn, 128x32x8_64x16x1_8x4_8x4_2x2, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   2
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   2 x   4
-//       Threadblock:  16 x  64 x 16
-CUTLASS_TEST_L2(SM50_device_cgemm_nn, 16x64x16_8x16x1_2x2_4x8_2x4, {
-    using precision = cutlass::complex<float>;
-    using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   2 x   4
-//       Threadblock:  16 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_cgemm_nn, 16x128x16_8x32x1_2x4_4x8_2x4, {
-    using precision = cutlass::complex<float>;
-    using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   8 x   4
@@ -1093,96 +1033,6 @@ CUTLASS_TEST_L1(SM50_device_cgemm_nn, 256x32x8_64x16x1_8x4_8x4_4x2, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   2
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   4 x   4
-//       Threadblock:  32 x  64 x 16
-CUTLASS_TEST_L2(SM50_device_cgemm_nn, 32x64x16_8x16x1_2x2_4x8_4x4, {
-    using precision = cutlass::complex<float>;
-    using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   4 x   4
-//       Threadblock:  32 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_cgemm_nn, 32x128x16_8x32x1_2x4_4x8_4x4, {
-    using precision = cutlass::complex<float>;
-    using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   2
-//    Threads / Warp:   8 x   4
-//     Warps / Block:   4 x   4
-//       Threadblock:  64 x  32 x 16
-CUTLASS_TEST_L2(SM50_device_cgemm_nn, 64x32x16_16x8x1_2x2_8x4_4x4, {
-    using precision = cutlass::complex<float>;
-    using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   4 x   2
 //    Threads / Warp:   4 x   8
@@ -1243,36 +1093,6 @@ CUTLASS_TEST_L1(SM50_device_cgemm_nn, 64x128x8_16x32x1_4x4_4x8_4x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   4 x   2
-//    Threads / Warp:   8 x   4
-//     Warps / Block:   4 x   4
-//       Threadblock: 128 x  32 x 16
-CUTLASS_TEST_L2(SM50_device_cgemm_nn, 128x32x16_32x8x1_4x2_8x4_4x4, {
-    using precision = cutlass::complex<float>;
-    using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   4 x   4
 //    Threads / Warp:   8 x   4

test/unit/gemm/device/simt_cgemm_tn_sm50.cu

@@ -673,66 +673,6 @@ CUTLASS_TEST_L1(SM50_device_cgemm_tn, 128x32x8_64x16x1_8x4_8x4_2x2, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   2
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   2 x   4
-//       Threadblock:  16 x  64 x 16
-CUTLASS_TEST_L2(SM50_device_cgemm_tn, 16x64x16_8x16x1_2x2_4x8_2x4, {
-    using precision = cutlass::complex<float>;
-    using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   2 x   4
-//       Threadblock:  16 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_cgemm_tn, 16x128x16_8x32x1_2x4_4x8_2x4, {
-    using precision = cutlass::complex<float>;
-    using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   8 x   4
@@ -1093,96 +1033,6 @@ CUTLASS_TEST_L1(SM50_device_cgemm_tn, 256x32x8_64x16x1_8x4_8x4_4x2, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   2
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   4 x   4
-//       Threadblock:  32 x  64 x 16
-CUTLASS_TEST_L2(SM50_device_cgemm_tn, 32x64x16_8x16x1_2x2_4x8_4x4, {
-    using precision = cutlass::complex<float>;
-    using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   4 x   4
-//       Threadblock:  32 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_cgemm_tn, 32x128x16_8x32x1_2x4_4x8_4x4, {
-    using precision = cutlass::complex<float>;
-    using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   2
-//    Threads / Warp:   8 x   4
-//     Warps / Block:   4 x   4
-//       Threadblock:  64 x  32 x 16
-CUTLASS_TEST_L2(SM50_device_cgemm_tn, 64x32x16_16x8x1_2x2_8x4_4x4, {
-    using precision = cutlass::complex<float>;
-    using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   4 x   2
 //    Threads / Warp:   4 x   8
@@ -1243,36 +1093,6 @@ CUTLASS_TEST_L1(SM50_device_cgemm_tn, 64x128x8_16x32x1_4x4_4x8_4x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   4 x   2
-//    Threads / Warp:   8 x   4
-//     Warps / Block:   4 x   4
-//       Threadblock: 128 x  32 x 16
-CUTLASS_TEST_L2(SM50_device_cgemm_tn, 128x32x16_32x8x1_4x2_8x4_4x4, {
-    using precision = cutlass::complex<float>;
-    using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   4 x   4
 //    Threads / Warp:   8 x   4

test/unit/gemm/device/simt_cgemm_tt_sm50.cu

@@ -673,66 +673,6 @@ CUTLASS_TEST_L1(SM50_device_cgemm_tt, 128x32x8_64x16x1_8x4_8x4_2x2, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   2
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   2 x   4
-//       Threadblock:  16 x  64 x 16
-CUTLASS_TEST_L2(SM50_device_cgemm_tt, 16x64x16_8x16x1_2x2_4x8_2x4, {
-    using precision = cutlass::complex<float>;
-    using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   2 x   4
-//       Threadblock:  16 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_cgemm_tt, 16x128x16_8x32x1_2x4_4x8_2x4, {
-    using precision = cutlass::complex<float>;
-    using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   8 x   4
@@ -1093,96 +1033,6 @@ CUTLASS_TEST_L1(SM50_device_cgemm_tt, 256x32x8_64x16x1_8x4_8x4_4x2, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   2
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   4 x   4
-//       Threadblock:  32 x  64 x 16
-CUTLASS_TEST_L2(SM50_device_cgemm_tt, 32x64x16_8x16x1_2x2_4x8_4x4, {
-    using precision = cutlass::complex<float>;
-    using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   4 x   4
-//       Threadblock:  32 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_cgemm_tt, 32x128x16_8x32x1_2x4_4x8_4x4, {
-    using precision = cutlass::complex<float>;
-    using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   2
-//    Threads / Warp:   8 x   4
-//     Warps / Block:   4 x   4
-//       Threadblock:  64 x  32 x 16
-CUTLASS_TEST_L2(SM50_device_cgemm_tt, 64x32x16_16x8x1_2x2_8x4_4x4, {
-    using precision = cutlass::complex<float>;
-    using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   4 x   2
 //    Threads / Warp:   4 x   8
@@ -1243,36 +1093,6 @@ CUTLASS_TEST_L1(SM50_device_cgemm_tt, 64x128x8_16x32x1_4x4_4x8_4x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   4 x   2
-//    Threads / Warp:   8 x   4
-//     Warps / Block:   4 x   4
-//       Threadblock: 128 x  32 x 16
-CUTLASS_TEST_L2(SM50_device_cgemm_tt, 128x32x16_32x8x1_4x2_8x4_4x4, {
-    using precision = cutlass::complex<float>;
-    using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   4 x   4
 //    Threads / Warp:   8 x   4
@@ -1302,4 +1122,3 @@ CUTLASS_TEST_L2(SM50_device_cgemm_tt, 128x64x8_32x16x1_4x4_8x4_4x4, {
     >;
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
-

test/unit/gemm/device/simt_dgemm_nn_sm50.cu

@@ -643,6 +643,45 @@ CUTLASS_TEST_L0(SM50_device_dgemm_nn, 64x64x8_32x32x1_8x4_4x8_2x2, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   8 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  64 x  64 x  8
+CUTLASS_TEST_L0(SM50_device_dgemm_affin2_nn, 64x64x8_32x32x1_8x4_4x8_2x2, {
+    using precision = double;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using LayoutA = cutlass::layout::AffineRank2ColumnMajor;
+    using LayoutB = cutlass::layout::AffineRank2ColumnMajor;
+    using LayoutC = cutlass::layout::AffineRankN<2>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, LayoutA,
+        precision, LayoutB,
+        precision, LayoutC,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+
+    typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
+    typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
+    typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
+
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
+} )
+
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   8 x   4
 //    Threads / Warp:   8 x   4
@@ -673,66 +712,6 @@ CUTLASS_TEST_L1(SM50_device_dgemm_nn, 128x32x8_64x16x1_8x4_8x4_2x2, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   2
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   2 x   4
-//       Threadblock:  16 x  64 x 16
-CUTLASS_TEST_L2(SM50_device_dgemm_nn, 16x64x16_8x16x1_2x2_4x8_2x4, {
-    using precision = double;
-    using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   2 x   4
-//       Threadblock:  16 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_dgemm_nn, 16x128x16_8x32x1_2x4_4x8_2x4, {
-    using precision = double;
-    using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   8 x   4
@@ -973,96 +952,6 @@ CUTLASS_TEST_L2(SM50_device_dgemm_nn, 128x32x8_32x16x1_4x4_8x4_4x2, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   2
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   4 x   4
-//       Threadblock:  32 x  64 x 16
-CUTLASS_TEST_L2(SM50_device_dgemm_nn, 32x64x16_8x16x1_2x2_4x8_4x4, {
-    using precision = double;
-    using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   4 x   4
-//       Threadblock:  32 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_dgemm_nn, 32x128x16_8x32x1_2x4_4x8_4x4, {
-    using precision = double;
-    using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   2
-//    Threads / Warp:   8 x   4
-//     Warps / Block:   4 x   4
-//       Threadblock:  64 x  32 x 16
-CUTLASS_TEST_L2(SM50_device_dgemm_nn, 64x32x16_16x8x1_2x2_8x4_4x4, {
-    using precision = double;
-    using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   4 x   2
 //    Threads / Warp:   4 x   8
@@ -1094,32 +983,3 @@ CUTLASS_TEST_L2(SM50_device_dgemm_nn, 64x64x8_16x16x1_4x2_4x8_4x4, {
 } )
 
 ////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   4 x   2
-//    Threads / Warp:   8 x   4
-//     Warps / Block:   4 x   4
-//       Threadblock: 128 x  32 x 16
-CUTLASS_TEST_L2(SM50_device_dgemm_nn, 128x32x16_32x8x1_4x2_8x4_4x4, {
-    using precision = double;
-    using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-

test/unit/gemm/device/simt_dgemm_nt_sm50.cu

@@ -643,6 +643,45 @@ CUTLASS_TEST_L0(SM50_device_dgemm_nt, 64x64x8_32x32x1_8x4_4x8_2x2, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   8 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  64 x  64 x  8
+CUTLASS_TEST_L0(SM50_device_dgemm_affine2_nt, 64x64x8_32x32x1_8x4_4x8_2x2, {
+    using precision = double;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using LayoutA = cutlass::layout::AffineRank2ColumnMajor;
+    using LayoutB = cutlass::layout::AffineRank2RowMajor;
+    using LayoutC = cutlass::layout::AffineRankN<2>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, LayoutA,
+        precision, LayoutB,
+        precision, LayoutC,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+
+    typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
+    typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
+    typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
+
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
+} )
+
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   8 x   4
 //    Threads / Warp:   8 x   4

test/unit/gemm/device/simt_dgemm_tn_sm50.cu

@@ -643,6 +643,45 @@ CUTLASS_TEST_L0(SM50_device_dgemm_tn, 64x64x8_32x32x1_8x4_4x8_2x2, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   8 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  64 x  64 x  8
+CUTLASS_TEST_L0(SM50_device_dgemm_affine2_tn, 64x64x8_32x32x1_8x4_4x8_2x2, {
+    using precision = double;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using LayoutA = cutlass::layout::AffineRank2RowMajor;
+    using LayoutB = cutlass::layout::AffineRank2ColumnMajor;
+    using LayoutC = cutlass::layout::AffineRankN<2>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, LayoutA,
+        precision, LayoutB,
+        precision, LayoutC,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+
+    typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
+    typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
+    typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
+
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
+} )
+
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   8 x   4
 //    Threads / Warp:   8 x   4
@@ -673,66 +712,6 @@ CUTLASS_TEST_L1(SM50_device_dgemm_tn, 128x32x8_64x16x1_8x4_8x4_2x2, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   2
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   2 x   4
-//       Threadblock:  16 x  64 x 16
-CUTLASS_TEST_L2(SM50_device_dgemm_tn, 16x64x16_8x16x1_2x2_4x8_2x4, {
-    using precision = double;
-    using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   2 x   4
-//       Threadblock:  16 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_dgemm_tn, 16x128x16_8x32x1_2x4_4x8_2x4, {
-    using precision = double;
-    using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   8 x   4
@@ -973,96 +952,6 @@ CUTLASS_TEST_L2(SM50_device_dgemm_tn, 128x32x8_32x16x1_4x4_8x4_4x2, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   2
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   4 x   4
-//       Threadblock:  32 x  64 x 16
-CUTLASS_TEST_L2(SM50_device_dgemm_tn, 32x64x16_8x16x1_2x2_4x8_4x4, {
-    using precision = double;
-    using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   4 x   4
-//       Threadblock:  32 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_dgemm_tn, 32x128x16_8x32x1_2x4_4x8_4x4, {
-    using precision = double;
-    using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   2
-//    Threads / Warp:   8 x   4
-//     Warps / Block:   4 x   4
-//       Threadblock:  64 x  32 x 16
-CUTLASS_TEST_L2(SM50_device_dgemm_tn, 64x32x16_16x8x1_2x2_8x4_4x4, {
-    using precision = double;
-    using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   4 x   2
 //    Threads / Warp:   4 x   8
@@ -1094,32 +983,3 @@ CUTLASS_TEST_L2(SM50_device_dgemm_tn, 64x64x8_16x16x1_4x2_4x8_4x4, {
 } )
 
 ////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   4 x   2
-//    Threads / Warp:   8 x   4
-//     Warps / Block:   4 x   4
-//       Threadblock: 128 x  32 x 16
-CUTLASS_TEST_L2(SM50_device_dgemm_tn, 128x32x16_32x8x1_4x2_8x4_4x4, {
-    using precision = double;
-    using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-

test/unit/gemm/device/simt_dgemm_tt_sm50.cu

@@ -163,6 +163,45 @@ CUTLASS_TEST_L0(SM50_device_dgemm_tt, 32x32x8_32x32x1_8x4_4x8_1x1, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   8 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   1
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L0(SM50_device_dgemm_affine2_tt, 32x32x8_32x32x1_8x4_4x8_1x1, {
+    using precision = double;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using LayoutA = cutlass::layout::AffineRank2ColumnMajor;
+    using LayoutB = cutlass::layout::AffineRank2ColumnMajor;
+    using LayoutC = cutlass::layout::AffineRankN<2>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, LayoutA,
+        precision, LayoutB,
+        precision, LayoutC,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+
+    typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
+    typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
+    typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
+
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
+} )
+
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   4 x   8
@@ -673,66 +712,6 @@ CUTLASS_TEST_L1(SM50_device_dgemm_tt, 128x32x8_64x16x1_8x4_8x4_2x2, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   2
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   2 x   4
-//       Threadblock:  16 x  64 x 16
-CUTLASS_TEST_L2(SM50_device_dgemm_tt, 16x64x16_8x16x1_2x2_4x8_2x4, {
-    using precision = double;
-    using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   2 x   4
-//       Threadblock:  16 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_dgemm_tt, 16x128x16_8x32x1_2x4_4x8_2x4, {
-    using precision = double;
-    using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   8 x   4
@@ -973,96 +952,6 @@ CUTLASS_TEST_L2(SM50_device_dgemm_tt, 128x32x8_32x16x1_4x4_8x4_4x2, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   2
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   4 x   4
-//       Threadblock:  32 x  64 x 16
-CUTLASS_TEST_L2(SM50_device_dgemm_tt, 32x64x16_8x16x1_2x2_4x8_4x4, {
-    using precision = double;
-    using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   4 x   4
-//       Threadblock:  32 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_dgemm_tt, 32x128x16_8x32x1_2x4_4x8_4x4, {
-    using precision = double;
-    using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   2
-//    Threads / Warp:   8 x   4
-//     Warps / Block:   4 x   4
-//       Threadblock:  64 x  32 x 16
-CUTLASS_TEST_L2(SM50_device_dgemm_tt, 64x32x16_16x8x1_2x2_8x4_4x4, {
-    using precision = double;
-    using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   4 x   2
 //    Threads / Warp:   4 x   8
@@ -1094,32 +983,3 @@ CUTLASS_TEST_L2(SM50_device_dgemm_tt, 64x64x8_16x16x1_4x2_4x8_4x4, {
 } )
 
 ////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   4 x   2
-//    Threads / Warp:   8 x   4
-//     Warps / Block:   4 x   4
-//       Threadblock: 128 x  32 x 16
-CUTLASS_TEST_L2(SM50_device_dgemm_tt, 128x32x16_32x8x1_4x2_8x4_4x4, {
-    using precision = double;
-    using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-

test/unit/gemm/device/simt_igemm_nn_sm50.cu

@@ -943,36 +943,6 @@ CUTLASS_TEST_L2(SM50_device_igemm_nn, 16x64x16_8x16x1_2x2_4x8_2x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   2 x   4
-//       Threadblock:  16 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_igemm_nn, 16x128x16_8x32x1_2x4_4x8_2x4, {
-    using precision = int;
-    using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   8 x   4
@@ -1483,36 +1453,6 @@ CUTLASS_TEST_L2(SM50_device_igemm_nn, 32x64x16_8x16x1_2x2_4x8_4x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   4 x   4
-//       Threadblock:  32 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_igemm_nn, 32x128x16_8x32x1_2x4_4x8_4x4, {
-    using precision = int;
-    using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   8 x   4

test/unit/gemm/device/simt_igemm_tn_sm50.cu

@@ -943,36 +943,6 @@ CUTLASS_TEST_L2(SM50_device_igemm_tn, 16x64x16_8x16x1_2x2_4x8_2x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   2 x   4
-//       Threadblock:  16 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_igemm_tn, 16x128x16_8x32x1_2x4_4x8_2x4, {
-    using precision = int;
-    using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   8 x   4
@@ -1483,36 +1453,6 @@ CUTLASS_TEST_L2(SM50_device_igemm_tn, 32x64x16_8x16x1_2x2_4x8_4x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   4 x   4
-//       Threadblock:  32 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_igemm_tn, 32x128x16_8x32x1_2x4_4x8_4x4, {
-    using precision = int;
-    using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   8 x   4
@@ -1633,36 +1573,6 @@ CUTLASS_TEST_L2(SM50_device_igemm_tn, 64x256x8_16x64x1_4x8_4x8_4x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   4 x   2
-//    Threads / Warp:   8 x   4
-//     Warps / Block:   4 x   4
-//       Threadblock: 128 x  32 x 16
-CUTLASS_TEST_L2(SM50_device_igemm_tn, 128x32x16_32x8x1_4x2_8x4_4x4, {
-    using precision = int;
-    using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   4 x   4
 //    Threads / Warp:   8 x   4

test/unit/gemm/device/simt_igemm_tt_sm50.cu

@@ -1633,36 +1633,6 @@ CUTLASS_TEST_L2(SM50_device_igemm_tt, 64x256x8_16x64x1_4x8_4x8_4x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   4 x   2
-//    Threads / Warp:   8 x   4
-//     Warps / Block:   4 x   4
-//       Threadblock: 128 x  32 x 16
-CUTLASS_TEST_L2(SM50_device_igemm_tt, 128x32x16_32x8x1_4x2_8x4_4x4, {
-    using precision = int;
-    using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   4 x   4
 //    Threads / Warp:   8 x   4

test/unit/gemm/device/simt_qgemm_nn_sm50.cu

@@ -0,0 +1,855 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+/*! \file
+    \brief Tests for device-wide GEMM interface
+*/
+
+#include <iostream>
+
+#include "cutlass/cutlass.h"
+#include "cutlass/gemm/device/gemm.h"
+#include "cutlass/numeric_types.h"
+
+#include "../../common/cutlass_unit_test.h"
+
+#include "cutlass/util/host_tensor.h"
+#include "cutlass/util/tensor_view_io.h"
+#include "cutlass/util/reference/host/tensor_fill.h"
+#include "cutlass/util/reference/host/tensor_copy.h"
+#include "cutlass/util/reference/host/tensor_compare.h"
+#include "cutlass/util/reference/host/gemm.h"
+
+#include "testbed.h"
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   1
+//       Threadblock:   8 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nn, 8x32x8_8x32x1_2x4_4x8_1x1, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<8, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   1
+//       Threadblock:  16 x  32 x  8
+CUTLASS_TEST_L0(SM50_device_qgemm_nn, 16x32x8_16x32x1_4x4_4x8_1x1, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   2
+//       Threadblock:   8 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nn, 8x32x8_8x16x1_2x2_4x8_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<8, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   2
+//       Threadblock:   8 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nn, 8x64x8_8x32x1_2x4_4x8_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<8, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   2
+//       Threadblock:  16 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nn, 16x32x8_16x16x1_4x2_4x8_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   2
+//       Threadblock:  16 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nn, 16x64x8_16x32x1_4x4_4x8_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   1 x   2
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nn, 32x32x8_32x16x1_4x4_8x4_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   1
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nn, 32x32x8_16x32x1_4x4_4x8_2x1, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  16 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nn, 16x32x8_8x16x1_2x2_4x8_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  16 x  64 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nn, 16x64x8_8x32x1_2x4_4x8_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nn, 32x32x8_16x16x1_4x2_4x8_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  32 x  64 x  8
+CUTLASS_TEST_L0(SM50_device_qgemm_nn, 32x64x8_16x32x1_4x4_4x8_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   2 x   2
+//       Threadblock:  64 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nn, 64x32x8_32x16x1_4x4_8x4_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   4
+//       Threadblock:  16 x  64 x 16
+CUTLASS_TEST_L2(SM50_device_qgemm_nn, 16x64x16_8x16x1_2x2_4x8_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   2 x   4
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nn, 32x32x8_16x8x1_2x2_8x4_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 8, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   4
+//       Threadblock:  32 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nn, 32x64x8_16x16x1_4x2_4x8_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   4
+//       Threadblock:  32 x 128 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nn, 32x128x8_16x32x1_4x4_4x8_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   2 x   4
+//       Threadblock:  64 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nn, 64x64x8_32x16x1_4x4_8x4_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   2
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nn, 32x32x8_8x16x1_2x2_4x8_4x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   2
+//       Threadblock:  64 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nn, 64x32x8_16x16x1_4x2_4x8_4x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   2
+//       Threadblock:  64 x  64 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nn, 64x64x8_16x32x1_4x4_4x8_4x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   4 x   2
+//       Threadblock: 128 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nn, 128x32x8_32x16x1_4x4_8x4_4x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   4
+//       Threadblock:  32 x  64 x 16
+CUTLASS_TEST_L2(SM50_device_qgemm_nn, 32x64x16_8x16x1_2x2_4x8_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   4 x   4
+//       Threadblock:  64 x  32 x 16
+CUTLASS_TEST_L2(SM50_device_qgemm_nn, 64x32x16_16x8x1_2x2_8x4_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   4
+//       Threadblock:  64 x  64 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nn, 64x64x8_16x16x1_4x2_4x8_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   4
+//       Threadblock:  64 x 128 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nn, 64x128x8_16x32x1_4x4_4x8_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 128, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   4 x   4
+//       Threadblock: 128 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nn, 128x64x8_32x16x1_4x4_8x4_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<128, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+

test/unit/gemm/device/simt_qgemm_nt_sm50.cu

@@ -0,0 +1,855 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+/*! \file
+    \brief Tests for device-wide GEMM interface
+*/
+
+#include <iostream>
+
+#include "cutlass/cutlass.h"
+#include "cutlass/gemm/device/gemm.h"
+#include "cutlass/numeric_types.h"
+
+#include "../../common/cutlass_unit_test.h"
+
+#include "cutlass/util/host_tensor.h"
+#include "cutlass/util/tensor_view_io.h"
+#include "cutlass/util/reference/host/tensor_fill.h"
+#include "cutlass/util/reference/host/tensor_copy.h"
+#include "cutlass/util/reference/host/tensor_compare.h"
+#include "cutlass/util/reference/host/gemm.h"
+
+#include "testbed.h"
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   1
+//       Threadblock:   8 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nt, 8x32x8_8x32x1_2x4_4x8_1x1, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<8, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   1
+//       Threadblock:  16 x  32 x  8
+CUTLASS_TEST_L0(SM50_device_qgemm_nt, 16x32x8_16x32x1_4x4_4x8_1x1, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   2
+//       Threadblock:   8 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nt, 8x32x8_8x16x1_2x2_4x8_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<8, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   2
+//       Threadblock:   8 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nt, 8x64x8_8x32x1_2x4_4x8_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<8, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   2
+//       Threadblock:  16 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nt, 16x32x8_16x16x1_4x2_4x8_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   2
+//       Threadblock:  16 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nt, 16x64x8_16x32x1_4x4_4x8_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   1 x   2
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nt, 32x32x8_32x16x1_4x4_8x4_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   1
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nt, 32x32x8_16x32x1_4x4_4x8_2x1, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  16 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nt, 16x32x8_8x16x1_2x2_4x8_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  16 x  64 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nt, 16x64x8_8x32x1_2x4_4x8_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nt, 32x32x8_16x16x1_4x2_4x8_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  32 x  64 x  8
+CUTLASS_TEST_L0(SM50_device_qgemm_nt, 32x64x8_16x32x1_4x4_4x8_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   2 x   2
+//       Threadblock:  64 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nt, 64x32x8_32x16x1_4x4_8x4_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   4
+//       Threadblock:  16 x  64 x 16
+CUTLASS_TEST_L2(SM50_device_qgemm_nt, 16x64x16_8x16x1_2x2_4x8_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   2 x   4
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nt, 32x32x8_16x8x1_2x2_8x4_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 8, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   4
+//       Threadblock:  32 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nt, 32x64x8_16x16x1_4x2_4x8_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   4
+//       Threadblock:  32 x 128 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nt, 32x128x8_16x32x1_4x4_4x8_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   2 x   4
+//       Threadblock:  64 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nt, 64x64x8_32x16x1_4x4_8x4_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   2
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nt, 32x32x8_8x16x1_2x2_4x8_4x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   2
+//       Threadblock:  64 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nt, 64x32x8_16x16x1_4x2_4x8_4x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   2
+//       Threadblock:  64 x  64 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nt, 64x64x8_16x32x1_4x4_4x8_4x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   4 x   2
+//       Threadblock: 128 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nt, 128x32x8_32x16x1_4x4_8x4_4x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   4
+//       Threadblock:  32 x  64 x 16
+CUTLASS_TEST_L2(SM50_device_qgemm_nt, 32x64x16_8x16x1_2x2_4x8_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   4 x   4
+//       Threadblock:  64 x  32 x 16
+CUTLASS_TEST_L2(SM50_device_qgemm_nt, 64x32x16_16x8x1_2x2_8x4_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   4
+//       Threadblock:  64 x  64 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_nt, 64x64x8_16x16x1_4x2_4x8_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   4
+//       Threadblock:  64 x 128 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nt, 64x128x8_16x32x1_4x4_4x8_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 128, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   4 x   4
+//       Threadblock: 128 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_nt, 128x64x8_32x16x1_4x4_8x4_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<128, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+

test/unit/gemm/device/simt_qgemm_tn_sm50.cu

@@ -0,0 +1,855 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+/*! \file
+    \brief Tests for device-wide GEMM interface
+*/
+
+#include <iostream>
+
+#include "cutlass/cutlass.h"
+#include "cutlass/gemm/device/gemm.h"
+#include "cutlass/numeric_types.h"
+
+#include "../../common/cutlass_unit_test.h"
+
+#include "cutlass/util/host_tensor.h"
+#include "cutlass/util/tensor_view_io.h"
+#include "cutlass/util/reference/host/tensor_fill.h"
+#include "cutlass/util/reference/host/tensor_copy.h"
+#include "cutlass/util/reference/host/tensor_compare.h"
+#include "cutlass/util/reference/host/gemm.h"
+
+#include "testbed.h"
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   1
+//       Threadblock:   8 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tn, 8x32x8_8x32x1_2x4_4x8_1x1, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<8, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   1
+//       Threadblock:  16 x  32 x  8
+CUTLASS_TEST_L0(SM50_device_qgemm_tn, 16x32x8_16x32x1_4x4_4x8_1x1, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   2
+//       Threadblock:   8 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tn, 8x32x8_8x16x1_2x2_4x8_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<8, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   2
+//       Threadblock:   8 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tn, 8x64x8_8x32x1_2x4_4x8_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<8, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   2
+//       Threadblock:  16 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tn, 16x32x8_16x16x1_4x2_4x8_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   2
+//       Threadblock:  16 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tn, 16x64x8_16x32x1_4x4_4x8_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   1 x   2
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tn, 32x32x8_32x16x1_4x4_8x4_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   1
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tn, 32x32x8_16x32x1_4x4_4x8_2x1, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  16 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tn, 16x32x8_8x16x1_2x2_4x8_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  16 x  64 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tn, 16x64x8_8x32x1_2x4_4x8_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tn, 32x32x8_16x16x1_4x2_4x8_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  32 x  64 x  8
+CUTLASS_TEST_L0(SM50_device_qgemm_tn, 32x64x8_16x32x1_4x4_4x8_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   2 x   2
+//       Threadblock:  64 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tn, 64x32x8_32x16x1_4x4_8x4_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   4
+//       Threadblock:  16 x  64 x 16
+CUTLASS_TEST_L2(SM50_device_qgemm_tn, 16x64x16_8x16x1_2x2_4x8_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   2 x   4
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tn, 32x32x8_16x8x1_2x2_8x4_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 8, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   4
+//       Threadblock:  32 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tn, 32x64x8_16x16x1_4x2_4x8_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   4
+//       Threadblock:  32 x 128 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tn, 32x128x8_16x32x1_4x4_4x8_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   2 x   4
+//       Threadblock:  64 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tn, 64x64x8_32x16x1_4x4_8x4_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   2
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tn, 32x32x8_8x16x1_2x2_4x8_4x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   2
+//       Threadblock:  64 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tn, 64x32x8_16x16x1_4x2_4x8_4x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   2
+//       Threadblock:  64 x  64 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tn, 64x64x8_16x32x1_4x4_4x8_4x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   4 x   2
+//       Threadblock: 128 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tn, 128x32x8_32x16x1_4x4_8x4_4x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   4
+//       Threadblock:  32 x  64 x 16
+CUTLASS_TEST_L2(SM50_device_qgemm_tn, 32x64x16_8x16x1_2x2_4x8_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   4 x   4
+//       Threadblock:  64 x  32 x 16
+CUTLASS_TEST_L2(SM50_device_qgemm_tn, 64x32x16_16x8x1_2x2_8x4_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   4
+//       Threadblock:  64 x  64 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tn, 64x64x8_16x16x1_4x2_4x8_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   4
+//       Threadblock:  64 x 128 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tn, 64x128x8_16x32x1_4x4_4x8_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 128, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   4 x   4
+//       Threadblock: 128 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tn, 128x64x8_32x16x1_4x4_8x4_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<128, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::ColumnMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+

test/unit/gemm/device/simt_qgemm_tt_sm50.cu

@@ -0,0 +1,855 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+/*! \file
+    \brief Tests for device-wide GEMM interface
+*/
+
+#include <iostream>
+
+#include "cutlass/cutlass.h"
+#include "cutlass/gemm/device/gemm.h"
+#include "cutlass/numeric_types.h"
+
+#include "../../common/cutlass_unit_test.h"
+
+#include "cutlass/util/host_tensor.h"
+#include "cutlass/util/tensor_view_io.h"
+#include "cutlass/util/reference/host/tensor_fill.h"
+#include "cutlass/util/reference/host/tensor_copy.h"
+#include "cutlass/util/reference/host/tensor_compare.h"
+#include "cutlass/util/reference/host/gemm.h"
+
+#include "testbed.h"
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   1
+//       Threadblock:   8 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tt, 8x32x8_8x32x1_2x4_4x8_1x1, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<8, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   1
+//       Threadblock:  16 x  32 x  8
+CUTLASS_TEST_L0(SM50_device_qgemm_tt, 16x32x8_16x32x1_4x4_4x8_1x1, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   2
+//       Threadblock:   8 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tt, 8x32x8_8x16x1_2x2_4x8_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<8, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   2
+//       Threadblock:   8 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tt, 8x64x8_8x32x1_2x4_4x8_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<8, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   2
+//       Threadblock:  16 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tt, 16x32x8_16x16x1_4x2_4x8_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   1 x   2
+//       Threadblock:  16 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tt, 16x64x8_16x32x1_4x4_4x8_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   1 x   2
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tt, 32x32x8_32x16x1_4x4_8x4_1x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   1
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tt, 32x32x8_16x32x1_4x4_4x8_2x1, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  16 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tt, 16x32x8_8x16x1_2x2_4x8_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  16 x  64 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tt, 16x64x8_8x32x1_2x4_4x8_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tt, 32x32x8_16x16x1_4x2_4x8_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   2
+//       Threadblock:  32 x  64 x  8
+CUTLASS_TEST_L0(SM50_device_qgemm_tt, 32x64x8_16x32x1_4x4_4x8_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   2 x   2
+//       Threadblock:  64 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tt, 64x32x8_32x16x1_4x4_8x4_2x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   4
+//       Threadblock:  16 x  64 x 16
+CUTLASS_TEST_L2(SM50_device_qgemm_tt, 16x64x16_8x16x1_2x2_4x8_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   2 x   4
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tt, 32x32x8_16x8x1_2x2_8x4_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 8, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   4
+//       Threadblock:  32 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tt, 32x64x8_16x16x1_4x2_4x8_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   2 x   4
+//       Threadblock:  32 x 128 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tt, 32x128x8_16x32x1_4x4_4x8_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   2 x   4
+//       Threadblock:  64 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tt, 64x64x8_32x16x1_4x4_8x4_2x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   2
+//       Threadblock:  32 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tt, 32x32x8_8x16x1_2x2_4x8_4x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   2
+//       Threadblock:  64 x  32 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tt, 64x32x8_16x16x1_4x2_4x8_4x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   2
+//       Threadblock:  64 x  64 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tt, 64x64x8_16x32x1_4x4_4x8_4x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   4 x   2
+//       Threadblock: 128 x  32 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tt, 128x32x8_32x16x1_4x4_8x4_4x2, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   4
+//       Threadblock:  32 x  64 x 16
+CUTLASS_TEST_L2(SM50_device_qgemm_tt, 32x64x16_8x16x1_2x2_4x8_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
+    using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   2 x   2
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   4 x   4
+//       Threadblock:  64 x  32 x 16
+CUTLASS_TEST_L2(SM50_device_qgemm_tt, 64x32x16_16x8x1_2x2_8x4_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   2
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   4
+//       Threadblock:  64 x  64 x  8
+CUTLASS_TEST_L2(SM50_device_qgemm_tt, 64x64x8_16x16x1_4x2_4x8_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   4 x   8
+//     Warps / Block:   4 x   4
+//       Threadblock:  64 x 128 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tt, 64x128x8_16x32x1_4x4_4x8_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<64, 128, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   4 x   4
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   4 x   4
+//       Threadblock: 128 x  64 x  8
+CUTLASS_TEST_L1(SM50_device_qgemm_tt, 128x64x8_32x16x1_4x4_8x4_4x4, {
+    using precision = cutlass::Quaternion<float>;
+    using ThreadblockShape = cutlass::gemm::GemmShape<128, 64, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision, cutlass::layout::RowMajor,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
+} )
+

test/unit/gemm/device/simt_sgemm_nn_sm50.cu

@@ -943,36 +943,6 @@ CUTLASS_TEST_L2(SM50_device_sgemm_nn, 16x64x16_8x16x1_2x2_4x8_2x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   2 x   4
-//       Threadblock:  16 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_sgemm_nn, 16x128x16_8x32x1_2x4_4x8_2x4, {
-    using precision = float;
-    using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   8 x   4
@@ -1213,6 +1183,45 @@ CUTLASS_TEST_L0(SM50_device_sgemm_nn, 128x128x8_64x32x1_8x8_8x4_2x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   8 x   8
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   2 x   4
+//       Threadblock: 128 x 128 x  8
+CUTLASS_TEST_L0(SM50_device_sgemm_affine2_nn, 128x128x8_64x32x1_8x8_8x4_2x4, {
+    using precision = float;
+    using ThreadblockShape = cutlass::gemm::GemmShape<128, 128, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<64, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using LayoutA = cutlass::layout::AffineRank2ColumnMajor;
+    using LayoutB = cutlass::layout::AffineRank2ColumnMajor;
+    using LayoutC = cutlass::layout::AffineRankN<2>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, LayoutA,
+        precision, LayoutB,
+        precision, LayoutC,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+
+    typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
+    typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
+    typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
+
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
+} )
+
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   4 x   8
@@ -1483,36 +1492,6 @@ CUTLASS_TEST_L2(SM50_device_sgemm_nn, 32x64x16_8x16x1_2x2_4x8_4x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   4 x   4
-//       Threadblock:  32 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_sgemm_nn, 32x128x16_8x32x1_2x4_4x8_4x4, {
-    using precision = float;
-    using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   8 x   4

test/unit/gemm/device/simt_sgemm_nt_sm50.cu

@@ -1213,6 +1213,45 @@ CUTLASS_TEST_L0(SM50_device_sgemm_nt, 128x128x8_64x32x1_8x8_8x4_2x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   8 x   8
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   2 x   4
+//       Threadblock: 128 x 128 x  8
+CUTLASS_TEST_L0(SM50_device_sgemm_affine2_nt, 128x128x8_64x32x1_8x8_8x4_2x4, {
+    using precision = float;
+    using ThreadblockShape = cutlass::gemm::GemmShape<128, 128, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<64, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using LayoutA = cutlass::layout::AffineRank2ColumnMajor;
+    using LayoutB = cutlass::layout::AffineRank2RowMajor;
+    using LayoutC = cutlass::layout::AffineRankN<2>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, LayoutA,
+        precision, LayoutB,
+        precision, LayoutC,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+
+    typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
+    typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
+    typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
+
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
+} )
+
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   4 x   8

test/unit/gemm/device/simt_sgemm_nt_sm80.cu

@@ -130,6 +130,43 @@ TEST(SM80_Device_Gemm_f32n_f32t_f32t_simt_f32, 128x128x8_32x64x1) {
   EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 }
 
+TEST(SM80_Device_Gemm_f32an_f32at_f32at_simt_f32, 128x128x8_32x64x1) {
+  
+  using Element = float;
+  using LayoutA = cutlass::layout::AffineRank2ColumnMajor;
+  using LayoutB = cutlass::layout::AffineRank2RowMajor;
+  using LayoutC = cutlass::layout::AffineRankN<2>;
+
+  using Gemm = cutlass::gemm::device::Gemm<
+    Element, 
+    LayoutA,
+    Element, 
+    LayoutB,
+    Element,
+    LayoutC, 
+    Element,
+    cutlass::arch::OpClassSimt, 
+    cutlass::arch::Sm80,
+    cutlass::gemm::GemmShape<128, 128, 8>,
+    cutlass::gemm::GemmShape<32, 64, 8>, 
+    cutlass::gemm::GemmShape<1, 1, 1>,
+    cutlass::epilogue::thread::LinearCombination<
+        Element, 
+        1,
+        Element, 
+        Element>,
+    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 
+    3
+  >;
+
+  typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
+  typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
+  typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
+
+  EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C ));
+
+}
+
 TEST(SM80_Device_Gemm_f32n_f32t_f32t_simt_f32, 64x128x8_32x64x1) {
   
   using Element = float;
@@ -248,7 +285,6 @@ TEST(SM80_Device_Gemm_f32n_f32t_f32t_simt_f32, 128x256x8_64x64x1) {
 }
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
-
 #endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
 
 /////////////////////////////////////////////////////////////////////////////////////////////////

test/unit/gemm/device/simt_sgemm_tn_sm50.cu

@@ -943,36 +943,6 @@ CUTLASS_TEST_L2(SM50_device_sgemm_tn, 16x64x16_8x16x1_2x2_4x8_2x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   2 x   4
-//       Threadblock:  16 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_sgemm_tn, 16x128x16_8x32x1_2x4_4x8_2x4, {
-    using precision = float;
-    using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   8 x   4
@@ -1213,6 +1183,45 @@ CUTLASS_TEST_L0(SM50_device_sgemm_tn, 128x128x8_64x32x1_8x8_8x4_2x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   8 x   8
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   2 x   4
+//       Threadblock: 128 x 128 x  8
+CUTLASS_TEST_L0(SM50_device_sgemm_affine2_tn, 128x128x8_64x32x1_8x8_8x4_2x4, {
+    using precision = float;
+    using ThreadblockShape = cutlass::gemm::GemmShape<128, 128, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<64, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using LayoutA = cutlass::layout::AffineRank2RowMajor;
+    using LayoutB = cutlass::layout::AffineRank2ColumnMajor;
+    using LayoutC = cutlass::layout::AffineRankN<2>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, LayoutA,
+        precision, LayoutB,
+        precision, LayoutC,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+
+    typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
+    typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
+    typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
+
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
+} )
+
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   4 x   8
@@ -1483,36 +1492,6 @@ CUTLASS_TEST_L2(SM50_device_sgemm_tn, 32x64x16_8x16x1_2x2_4x8_4x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   2 x   4
-//    Threads / Warp:   4 x   8
-//     Warps / Block:   4 x   4
-//       Threadblock:  32 x 128 x 16
-CUTLASS_TEST_L2(SM50_device_sgemm_tn, 32x128x16_8x32x1_2x4_4x8_4x4, {
-    using precision = float;
-    using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   8 x   4
@@ -1633,36 +1612,6 @@ CUTLASS_TEST_L2(SM50_device_sgemm_tn, 64x256x8_16x64x1_4x8_4x8_4x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   4 x   2
-//    Threads / Warp:   8 x   4
-//     Warps / Block:   4 x   4
-//       Threadblock: 128 x  32 x 16
-CUTLASS_TEST_L2(SM50_device_sgemm_tn, 128x32x16_32x8x1_4x2_8x4_4x4, {
-    using precision = float;
-    using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::ColumnMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   4 x   4
 //    Threads / Warp:   8 x   4

test/unit/gemm/device/simt_sgemm_tn_sm80.cu

@@ -44,7 +44,6 @@
     
 #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
 ////////////////////////////////////////////////////////////////////////////////
-    
 TEST(SM80_Device_Gemm_f32t_f32n_f32t_simt_f32, 32x64x8_32x64x1) {
   
   using Element = float;
@@ -132,6 +131,42 @@ TEST(SM80_Device_Gemm_f32t_f32n_f32t_simt_f32, 128x128x8_32x64x1) {
   EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 }
 
+TEST(SM80_Device_Gemm_f32at_f32an_f32t_simt_f32, 128x128x8_32x64x1) {
+  
+  using Element = float;
+  using LayoutA = cutlass::layout::AffineRank2RowMajor;
+  using LayoutB = cutlass::layout::AffineRank2ColumnMajor;
+  using LayoutC = cutlass::layout::RowMajor;
+
+  using Gemm = cutlass::gemm::device::Gemm<
+    Element, 
+    LayoutA,
+    Element, 
+    LayoutB,
+    Element,
+    LayoutC, 
+    Element,
+    cutlass::arch::OpClassSimt, 
+    cutlass::arch::Sm80,
+    cutlass::gemm::GemmShape<128, 128, 8>,
+    cutlass::gemm::GemmShape<32, 64, 8>, 
+    cutlass::gemm::GemmShape<1, 1, 1>,
+    cutlass::epilogue::thread::LinearCombination<
+        Element, 
+        1,
+        Element, 
+        Element>,
+    cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 
+    3
+  >;
+
+  typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
+  typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
+  typename LayoutC::Stride::Index stride_factor_C[] = {1};
+
+  EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>( stride_factor_A, stride_factor_B, stride_factor_C ));
+}
+
 TEST(SM80_Device_Gemm_f32t_f32n_f32t_simt_f32, 64x128x8_32x64x1) {
   
   using Element = float;

test/unit/gemm/device/simt_sgemm_tt_sm50.cu

@@ -1213,6 +1213,45 @@ CUTLASS_TEST_L0(SM50_device_sgemm_tt, 128x128x8_64x32x1_8x8_8x4_2x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
+////////////////////////////////////////////////////////////////////////////////
+// Elements / Thread:   8 x   8
+//    Threads / Warp:   8 x   4
+//     Warps / Block:   2 x   4
+//       Threadblock: 128 x 128 x  8
+CUTLASS_TEST_L0(SM50_device_sgemm_affine2_tt, 128x128x8_64x32x1_8x8_8x4_2x4, {
+    using precision = float;
+    using ThreadblockShape = cutlass::gemm::GemmShape<128, 128, 8>;
+    using WarpShape = cutlass::gemm::GemmShape<64, 32, 8>;
+
+    static int const kEpilogueElementsPerAccess = 1;
+    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
+    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
+        precision, kEpilogueElementsPerAccess, precision, precision>;
+
+    using LayoutA = cutlass::layout::AffineRank2ColumnMajor;
+    using LayoutB = cutlass::layout::AffineRank2ColumnMajor;
+    using LayoutC = cutlass::layout::AffineRankN<2>;
+
+    using Gemm = cutlass::gemm::device::Gemm<
+        precision, LayoutA,
+        precision, LayoutB,
+        precision, LayoutC,
+        precision,
+        cutlass::arch::OpClassSimt,
+        cutlass::arch::Sm50,
+        ThreadblockShape, WarpShape, InstructionShape,
+        EpilogueOutputOp,
+        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
+        2 // Stages
+    >;
+
+    typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
+    typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
+    typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
+
+    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
+} )
+
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   2 x   2
 //    Threads / Warp:   4 x   8
@@ -1633,36 +1672,6 @@ CUTLASS_TEST_L2(SM50_device_sgemm_tt, 64x256x8_16x64x1_4x8_4x8_4x4, {
     EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 } )
 
-////////////////////////////////////////////////////////////////////////////////
-// Elements / Thread:   4 x   2
-//    Threads / Warp:   8 x   4
-//     Warps / Block:   4 x   4
-//       Threadblock: 128 x  32 x 16
-CUTLASS_TEST_L2(SM50_device_sgemm_tt, 128x32x16_32x8x1_4x2_8x4_4x4, {
-    using precision = float;
-    using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
-    using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
-
-    static int const kEpilogueElementsPerAccess = 1;
-    using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
-    using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
-        precision, kEpilogueElementsPerAccess, precision, precision>;
-
-    using Gemm = cutlass::gemm::device::Gemm<
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision, cutlass::layout::RowMajor,
-        precision,
-        cutlass::arch::OpClassSimt,
-        cutlass::arch::Sm50,
-        ThreadblockShape, WarpShape, InstructionShape,
-        EpilogueOutputOp,
-        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
-        2 // Stages
-    >;
-    EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
-} )
-
 ////////////////////////////////////////////////////////////////////////////////
 // Elements / Thread:   4 x   4
 //    Threads / Warp:   8 x   4

test/unit/gemm/device/simt_sm50.py

@@ -45,14 +45,15 @@ warpShapeMin = 8*8
 
 threadblockEdgeMax = 256
 
-#      char,      type             bits/elem, max tile,    L0 threadblock tiles
+#      char,      type               bits/elem, max tile,   L0 threadblock tiles
 precisions = [
-       ["c", "cutlass::complex<float>",   64,  64*128, [ [ 64, 128], [ 64,  32]             ] ],
-       ["d", "double",                    64,   64*64, [ [ 64,  64], [ 32,  32]             ] ],
-       ["h", "cutlass::half_t",           16, 128*256, [ [256, 128], [ 64, 128], [ 64,  32] ] ],
-       ["i", "int",                       32, 128*128, [ [128,  64], [ 16, 32]              ] ],
-       ["s", "float",                     32, 128*128, [ [128, 256], [128, 128], [ 64,  64] ] ],
-       ["z", "cutlass::complex<double>", 128,   64*64, [ [ 32,  64], [ 16,  32]             ] ],
+       ["c", "cutlass::complex<float>",     64,  64*128, [ [ 64, 128], [ 64,  32]             ] ],
+       ["q", "cutlass::Quaternion<float>",  64,  64*128, [ [ 64, 128], [ 64,  32]             ] ],
+       ["d", "double",                      64,   64*64, [ [ 64,  64], [ 32,  32]             ] ],
+       ["h", "cutlass::half_t",             16, 128*256, [ [256, 128], [ 64, 128], [ 64,  32] ] ],
+       ["i", "int",                         32, 128*128, [ [128,  64], [ 16, 32]              ] ],
+       ["s", "float",                       32, 128*128, [ [128, 256], [128, 128], [ 64,  64] ] ],
+       ["z", "cutlass::complex<double>",   128,   64*64, [ [ 32,  64], [ 16,  32]             ] ],
        ]
 # L1 will have a single kernel for every unique shape
 # L2 will have everything else
@@ -313,7 +314,7 @@ for precision in precisions:
                     "        cutlass::arch::Sm50,\n"
                     "        ThreadblockShape, WarpShape, InstructionShape,\n"
                     "        EpilogueOutputOp,\n"
-                    "        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,\n"
+                    "        cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,\n"
                     "        2 // Stages\n"
                     "    >;\n" % (
                         "Column" if columnMajorA else "Row",

test/unit/gemm/device/testbed.h

@@ -45,6 +45,8 @@
 
 #include "testbed_utils.h"
 
+#include "cutlass/layout/matrix.h"
+
 namespace test {
 namespace gemm {
 namespace device {
@@ -58,6 +60,9 @@ struct Testbed {
   using ElementCompute = typename Gemm::GemmKernel::Epilogue::OutputOp::ElementCompute;
 
   /// Initialization
+  typename Gemm::LayoutA::Stride stride_factor_A;
+  typename Gemm::LayoutB::Stride stride_factor_B;
+  typename Gemm::LayoutC::Stride stride_factor_C;
   cutlass::Distribution::Kind init_A;
   cutlass::Distribution::Kind init_B;
   cutlass::Distribution::Kind init_C;
@@ -79,6 +84,23 @@ struct Testbed {
     cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
     uint64_t seed_ = 2080
   ):
+    stride_factor_A(typename Gemm::LayoutA::Stride()),
+    stride_factor_B(typename Gemm::LayoutB::Stride()),
+    stride_factor_C(typename Gemm::LayoutC::Stride()),
+    init_A(init_A_), init_B(init_B_), init_C(init_C_), seed(seed_) { }
+
+  Testbed(
+    typename Gemm::LayoutA::Stride stride_factor_A_,
+    typename Gemm::LayoutB::Stride stride_factor_B_,
+    typename Gemm::LayoutC::Stride stride_factor_C_,
+    cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
+    cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
+    cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
+    uint64_t seed_ = 2080
+  ):
+    stride_factor_A(stride_factor_A_),
+    stride_factor_B(stride_factor_B_),
+    stride_factor_C(stride_factor_C_),
     init_A(init_A_), init_B(init_B_), init_C(init_C_), seed(seed_) { }
 
   /// Helper to initialize a tensor view
@@ -139,11 +161,11 @@ struct Testbed {
     // Allocate the GEMM workspace
     //
 
-    tensor_A.resize(problem_size.mk());
-    tensor_B.resize(problem_size.kn());
-    tensor_C.resize(problem_size.mn());
-    tensor_D.resize(problem_size.mn());
-    reference_D.resize(problem_size.mn(), false);
+    tensor_A.resize(problem_size.mk(), cutlass::layout::Affine2Layout_Factory<typename Gemm::LayoutA>::layout_factory(problem_size.mk(), stride_factor_A));
+    tensor_B.resize(problem_size.kn(), cutlass::layout::Affine2Layout_Factory<typename Gemm::LayoutB>::layout_factory(problem_size.kn(), stride_factor_B));
+    tensor_C.resize(problem_size.mn(), cutlass::layout::Affine2Layout_Factory<typename Gemm::LayoutC>::layout_factory(problem_size.mn(), stride_factor_C));
+    tensor_D.resize(problem_size.mn(), cutlass::layout::Affine2Layout_Factory<typename Gemm::LayoutC>::layout_factory(problem_size.mn(), stride_factor_C));
+    reference_D.resize(problem_size.mn(), cutlass::layout::Affine2Layout_Factory<typename Gemm::LayoutC>::layout_factory(problem_size.mn(), stride_factor_C), false);
 
     EXPECT_TRUE(initialize_tensor(tensor_A.host_view(), init_A, seed + 2019));
     EXPECT_TRUE(initialize_tensor(tensor_B.host_view(), init_B, seed + 2018));
@@ -153,7 +175,7 @@ struct Testbed {
     // in the upper left corner of each operand.
     tensor_A.host_view().at({0, 0}) = typename Gemm::ElementA(1);
     tensor_B.host_view().at({0, 0}) = typename Gemm::ElementB(1);
-    tensor_C.host_view().at({0, 0}) = typename Gemm::ElementC(1);
+    tensor_C.host_view().at(cutlass::make_Coord(0, 0)) = typename Gemm::ElementC(1);
 
     cutlass::reference::host::TensorCopy(reference_D.host_view(), tensor_C.host_view());
 
@@ -226,7 +248,7 @@ struct Testbed {
     //
     // Verify
     //
-
+    
     cutlass::reference::host::Gemm<
         typename Gemm::ElementA, typename Gemm::LayoutA,
         typename Gemm::ElementB, typename Gemm::LayoutB,
@@ -347,7 +369,10 @@ struct Testbed {
 /////////////////////////////////////////////////////////////////////////////////////////////////
 
 template <typename Gemm>
-bool TestAllGemm() {
+bool TestAllGemm(
+    const typename Gemm::LayoutA::Stride& stride_factor_A = typename Gemm::LayoutA::Stride(),
+    const typename Gemm::LayoutB::Stride& stride_factor_B = typename Gemm::LayoutB::Stride(),
+    const typename Gemm::LayoutC::Stride& stride_factor_C = typename Gemm::LayoutC::Stride()) {
   bool passed = true;
 
   int const kMinimumOperandElementSize = 
@@ -393,7 +418,7 @@ bool TestAllGemm() {
     2.0
   };
 
-  Testbed<Gemm> testbed;
+  Testbed<Gemm> testbed(stride_factor_A, stride_factor_B, stride_factor_C);
 
   using ElementCompute = typename Gemm::EpilogueOutputOp::ElementCompute;
 
@@ -414,7 +439,6 @@ bool TestAllGemm() {
             for (auto beta : problem_beta) {
 
               cutlass::gemm::GemmCoord problem_size(m, n, k);
-
               passed = testbed.run(
                 problem_size, 
                 split_k,

test/unit/gemm/device/testbed_gemm_with_broadcast.h

@@ -0,0 +1,651 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+/*! \file
+    \brief Tests for device-wide GEMM interface
+*/
+
+#pragma once
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+
+#include "../../common/cutlass_unit_test.h"
+
+#include "cutlass/util/host_tensor.h"
+#include "cutlass/util/tensor_view_io.h"
+#include "cutlass/util/distribution.h"
+#include "cutlass/util/reference/host/tensor_fill.h"
+#include "cutlass/util/reference/host/tensor_copy.h"
+#include "cutlass/util/reference/host/tensor_compare.h"
+#include "cutlass/util/reference/host/tensor_norm.h"
+#include "cutlass/util/reference/host/gemm.h"
+#include "cutlass/util/reference/host/gemm_complex.h"
+
+#include "testbed_utils.h"
+
+namespace test {
+namespace gemm {
+namespace device {
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename Gemm>
+struct GemmWithBroadcastReferenceOp {
+
+  using OutputOp = typename Gemm::GemmKernel::Epilogue::OutputOp;
+
+  using ElementCompute = typename OutputOp::ElementCompute;
+  using ElementZ = typename OutputOp::ElementZ;
+  using ElementT = typename OutputOp::ElementT;
+
+  typename OutputOp::BinaryOp binary_op;
+  typename OutputOp::ElementwiseOp elementwise_op;
+
+  GemmWithBroadcastReferenceOp() { }
+
+  void operator()(ElementZ &Z, ElementT &T, ElementCompute gemm, ElementCompute bias) {
+
+    ElementCompute z_full = binary_op(gemm, bias);
+    Z = ElementZ(z_full);
+
+    ElementCompute t_full = elementwise_op(z_full);
+    T = ElementT(t_full);
+  }
+};
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+// Fused testbed
+//
+//  Y = GEMM(AB, C)
+//
+//  Z[i, j] = ReductionOp(Y[i, j], Broadcast[i])
+//
+//  T[i, j] = Elementwise(Z[i, j])
+//
+
+template <
+  typename Gemm, 
+  typename ReferenceOp = GemmWithBroadcastReferenceOp<Gemm>
+>
+struct TestbedGemmWithBroadcast {
+
+  using OutputOp = typename Gemm::GemmKernel::Epilogue::OutputOp;
+  using ElementC = typename Gemm::ElementC;
+  using ElementAccumulator = typename Gemm::ElementAccumulator;
+  using ElementCOmpute = typename OutputOp::ElementCompute;
+  using ElementZ = typename OutputOp::ElementZ;
+  using ElementT = typename OutputOp::ElementT;
+
+
+  /// Initialization
+  cutlass::Distribution::Kind init_A;
+  cutlass::Distribution::Kind init_B;
+  cutlass::Distribution::Kind init_C;
+  uint64_t seed;
+
+  cutlass::HostTensor<typename Gemm::ElementA, typename Gemm::LayoutA> tensor_A;          // Input A
+  cutlass::HostTensor<typename Gemm::ElementB, typename Gemm::LayoutB> tensor_B;          // Input B
+  cutlass::HostTensor<ElementC, typename Gemm::LayoutC> tensor_C;                         // Input C
+  cutlass::HostTensor<ElementC, typename Gemm::LayoutC> tensor_Broadcast;                 // Input Broadcast
+
+  cutlass::HostTensor<ElementZ, typename Gemm::LayoutC> tensor_Z;
+  cutlass::HostTensor<ElementT, typename Gemm::LayoutC> tensor_T;
+
+  cutlass::HostTensor<ElementAccumulator, typename Gemm::LayoutC> tensor_C_ref;
+  cutlass::HostTensor<ElementAccumulator, typename Gemm::LayoutC> tensor_Y_ref;
+  cutlass::HostTensor<ElementZ, typename Gemm::LayoutC> tensor_Z_ref;
+  cutlass::HostTensor<ElementT, typename Gemm::LayoutC> tensor_T_ref;
+
+
+  //
+  // Methods
+  //
+
+  TestbedGemmWithBroadcast(
+    cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
+    cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
+    cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
+    uint64_t seed_ = 2080
+  ):
+    init_A(init_A_), init_B(init_B_), init_C(init_C_), seed(seed_) { }
+
+  /// Helper to initialize a tensor view
+  template <typename Element, typename Layout>
+  bool initialize_tensor(
+    cutlass::TensorView<Element, Layout> view, 
+    cutlass::Distribution::Kind dist_kind,
+    uint64_t seed) {
+
+    if (dist_kind == cutlass::Distribution::Uniform) {
+
+      double scope_max, scope_min;
+      int bits_input = cutlass::sizeof_bits<Element>::value;
+      int bits_output = cutlass::sizeof_bits<typename Gemm::ElementC>::value;
+
+      if (bits_input == 1) {
+        scope_max = 2;
+        scope_min = 0;
+      } else if (bits_input <= 8) {
+        scope_max = 2;
+        scope_min = -2;
+      } else if (bits_output == 16) {
+        scope_max = 5;
+        scope_min = -5;
+      } else {
+        scope_max = 8;
+        scope_min = -8;
+      }
+
+      cutlass::reference::host::TensorFillRandomUniform(
+        view, seed, scope_max, scope_min, 0);
+    } 
+    else if (dist_kind == cutlass::Distribution::Identity) {
+
+      cutlass::reference::host::TensorFillIdentity(view);
+    } 
+    else if (dist_kind == cutlass::Distribution::Gaussian) {
+
+      cutlass::reference::host::TensorFillRandomGaussian(view, seed, 0, 0.5);
+    }
+    else if (dist_kind == cutlass::Distribution::Sequential) {
+
+      cutlass::reference::host::BlockFillSequential(
+        view.data(), view.capacity());
+    } 
+    else {
+      // TODO: Implement the rest
+      EXPECT_TRUE(false) << "Not implemented";
+      return false;
+    }
+
+    return true;
+  }
+
+  /// Initializes data structures
+  void initialize(cutlass::gemm::GemmCoord problem_size) {
+    //
+    // Allocate the GEMM workspace
+    //
+
+    tensor_A.resize(problem_size.mk());
+    tensor_B.resize(problem_size.kn());
+    tensor_C.resize(problem_size.mn());
+    tensor_Z.resize(problem_size.mn());
+    tensor_T.resize(problem_size.mn());
+    tensor_Broadcast.resize({
+      problem_size.m(), 
+      1
+    });
+
+    tensor_C_ref.resize(problem_size.mn());
+    tensor_Y_ref.resize(problem_size.mn());
+    tensor_Z_ref.resize(problem_size.mn());
+    tensor_T_ref.resize(problem_size.mn());
+
+    EXPECT_TRUE(initialize_tensor(tensor_A.host_view(), init_A, seed + 2019));
+    EXPECT_TRUE(initialize_tensor(tensor_B.host_view(), init_B, seed + 2018));
+    EXPECT_TRUE(initialize_tensor(tensor_C.host_view(), init_C, seed + 2017));
+    EXPECT_TRUE(initialize_tensor(tensor_Broadcast.host_view(), init_C, seed + 2020));
+
+    // It is possible to randomly initialize to all zeros, so override this with non-zeros
+    // in the upper left corner of each operand.
+    tensor_A.host_view().at({0, 0}) = typename Gemm::ElementA(1);
+    tensor_B.host_view().at({0, 0}) = typename Gemm::ElementB(1);
+    tensor_C.host_view().at({0, 0}) = typename Gemm::ElementC(1);
+
+    for (int m = 0; m < tensor_C_ref.extent().row(); ++m) {
+      for (int n = 0; n < tensor_C_ref.extent().column(); ++n) {
+        tensor_C_ref.at({m, n}) = ElementAccumulator(tensor_C.at({m, n}));
+      }
+    }
+
+    tensor_A.sync_device();
+    tensor_B.sync_device();
+    tensor_C.sync_device();
+    tensor_Broadcast.sync_device();
+
+    tensor_Z.sync_device();
+    tensor_T.sync_device();
+  }
+
+  /// Compares computed reference with device reference and outputs to a file if incorrect
+  bool compare_reference(
+    cutlass::gemm::GemmCoord problem_size, 
+    ElementAccumulator alpha, 
+    ElementAccumulator beta) {
+
+    tensor_Z.sync_host();
+    tensor_T.sync_host();
+
+    EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_A.host_view()), 0);
+    EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_B.host_view()), 0);
+    EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_C.host_view()), 0);
+    
+    EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_Z.host_view()), 0);
+    EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_T.host_view()), 0);
+
+    EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_Z_ref.host_view()), 0);
+    EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_T_ref.host_view()), 0);
+
+    bool passed = true;
+    float norm_diff = 0;
+
+    if (OutputOp::kStoreZ) {
+      norm_diff = cutlass::reference::host::TensorNormDiff(tensor_Z_ref.host_view(), tensor_Z.host_view(), float());
+      passed = (norm_diff <= 0.1f);
+      EXPECT_LT(norm_diff, 0.1f) << " tensor_Z is incorrect";
+    }
+
+    if (OutputOp::kStoreT) {
+
+      norm_diff = cutlass::reference::host::TensorNormDiff(tensor_T_ref.host_view(), tensor_T.host_view(), float());
+      passed = (passed && (norm_diff <= 0.1f));
+
+      EXPECT_LT(norm_diff, 0.1f) << " tensor_T is incorrect"; 
+    }
+
+
+    if (!passed) {
+
+      /*
+      std::stringstream fname;
+
+      fname << "error_Gemm_device_"
+        << problem_size.m() << "x"
+        << problem_size.n() << "x"
+        << problem_size.k() << "_"
+        << Gemm::ThreadblockShape::kM << "x"  
+        << Gemm::ThreadblockShape::kN << "x"  
+        << Gemm::ThreadblockShape::kK << "_"
+        << Gemm::WarpShape::kM << "x"  
+        << Gemm::WarpShape::kN << "x"  
+        << Gemm::WarpShape::kK << ".txt";
+
+      std::ofstream file(fname.str());
+      */
+
+      std::ofstream file("errors_testbed_gemm_with_broadcast.txt");
+
+
+      file
+        << "problem: " << problem_size 
+        << ", alpha: " << alpha << ", beta: " << beta << "\n\n";
+
+      file 
+        << "A =\n" << tensor_A.host_view()
+        << "\nB =\n" << tensor_B.host_view()
+        << "\nC =\n" << tensor_C.host_view()
+        << "\nZ =\n" << tensor_Z.host_view()
+        << "\nT =\n" << tensor_T.host_view()
+        << "\n\n"
+        << "\nY_ref =\n" << tensor_Y_ref.host_view()
+        << "\nZ_ref =\n" << tensor_Z_ref.host_view()
+        << "\nT_ref =\n" << tensor_T_ref.host_view();
+    }
+
+    return passed;
+  }
+
+  /// Verifies the result is a GEMM
+  bool verify(
+    cutlass::gemm::GemmCoord problem_size, 
+    ElementAccumulator alpha, 
+    ElementAccumulator beta) {
+
+    //
+    // Verify
+    //
+
+    cutlass::reference::host::GemmComplex<
+        typename Gemm::ElementA, typename Gemm::LayoutA,
+        typename Gemm::ElementB, typename Gemm::LayoutB,
+        ElementAccumulator, typename Gemm::LayoutC, 
+        ElementAccumulator, ElementAccumulator
+    >(
+      problem_size,
+      alpha, 
+      tensor_A.host_ref(),
+      Gemm::kTransformA,
+      tensor_B.host_ref(),
+      Gemm::kTransformB,
+      beta, 
+      tensor_C_ref.host_ref(), 
+      tensor_Y_ref.host_ref(), 
+      ElementAccumulator(0)
+    );
+
+    using ElementC = typename Gemm::ElementC;
+
+    ReferenceOp reference_op;
+
+
+    // compute tensor Z and tensor T
+    for (int m = 0; m < problem_size.m(); ++m) {
+      for (int n = 0; n < problem_size.n(); ++n) {
+
+        ElementZ z;
+        ElementT t;
+
+        reference_op(z, t, tensor_Y_ref.at({m, n}), tensor_Broadcast.at({m, 0}));
+
+        tensor_Z_ref.at({m, n}) = z;
+        tensor_T_ref.at({m, n}) = t;
+      }
+    }
+
+    return compare_reference(problem_size, alpha, beta);
+  }
+
+  /// Returns true if the CUDA device is sufficient to execute the kernel.
+  bool sufficient() const {
+
+    //
+    // Determine SMEM requirements and waive if not satisfied
+    //
+
+    int smem_size = int(sizeof(typename Gemm::GemmKernel::SharedStorage));
+
+    cudaDeviceProp properties;
+    int device_idx;
+    cudaError_t result = cudaGetDevice(&device_idx);
+
+    if (result != cudaSuccess) {
+      throw std::runtime_error("cudaGetDevice() API call failed.");
+    }
+
+    result = cudaGetDeviceProperties(&properties, device_idx);
+
+    if (result != cudaSuccess) {
+      throw std::runtime_error("cudaGetDeviceProperties() failed");
+    }
+
+    if (properties.sharedMemPerMultiprocessor < smem_size) {
+      return false;
+    }
+
+    return true;
+  }
+
+  /// Executes one test
+  bool run(
+    cutlass::gemm::GemmUniversalMode mode,
+    cutlass::gemm::GemmCoord problem_size, 
+    int batch_count = 1,
+    ElementAccumulator alpha = ElementAccumulator(1), 
+    ElementAccumulator beta = ElementAccumulator(0)) {
+
+    // Waive test if insufficient CUDA device
+    if (!sufficient()) {
+      if (CUTLASS_TEST_UNIT_ENABLE_WARNINGS) {
+        std::cerr << "Test waived due to insufficient CUDA device." << std::endl;
+      }
+      return true;
+    }
+
+    this->initialize(problem_size);
+
+    //
+    // Initialize the GEMM operator
+    //
+
+    typename Gemm::Arguments arguments{
+      mode,
+      problem_size,
+      batch_count,
+      {alpha, beta},
+      tensor_A.device_data(),
+      tensor_B.device_data(),
+      tensor_C.device_data(),
+      tensor_Z.device_data(),
+      tensor_Broadcast.device_data(),
+      tensor_T.device_data(),
+      problem_size.m() * problem_size.k(),
+      problem_size.n() * problem_size.k(),
+      problem_size.m() * problem_size.n(),
+      problem_size.m() * problem_size.n(),
+      problem_size.m(),
+      problem_size.m() * problem_size.n(),
+      tensor_A.layout().stride(0),
+      tensor_B.layout().stride(0),
+      tensor_C.layout().stride(0),
+      tensor_Z.layout().stride(0),
+      0,                                    // This must be zero
+      tensor_T.layout().stride(0),
+    };
+
+    Gemm gemm_op;
+
+    size_t workspace_size = Gemm::get_workspace_size(arguments);
+
+    cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
+
+    cutlass::Status status = gemm_op.initialize(arguments, workspace.get());
+
+
+    EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
+
+    //
+    // Run the GEMM
+    //
+
+    status = gemm_op();
+
+    EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
+
+    //
+    // Verify
+    //
+
+    bool passed = true;
+
+    passed = this->verify(problem_size, alpha, beta);
+
+    if (!passed) {
+      std::cout << "Failed with batch_count/split_k_slices = " << batch_count << std::endl;
+    }
+
+    //
+    // Profile
+    //
+
+    #if 0 // profiling disabled for now.
+
+    int const kWorkspaces = 100;
+
+    cutlass::DeviceAllocation<typename Gemm::ElementA> profiling_tensor_A(tensor_A.capacity() * kWorkspaces);
+    cutlass::DeviceAllocation<typename Gemm::ElementB> profiling_tensor_B(tensor_B.capacity() * kWorkspaces);
+    cutlass::DeviceAllocation<ElementC> profiling_tensor_C(tensor_C.capacity() * kWorkspaces);
+    cutlass::DeviceAllocation<ElementC> profiling_tensor_Broadcast(tensor_Broadcast.capacity() * kWorkspaces);
+    cutlass::DeviceAllocation<ElementZ> profiling_tensor_Z(tensor_Z.capacity() * kWorkspaces);
+    cutlass::DeviceAllocation<ElementT> profiling_tensor_T(tensor_T.capacity() * kWorkspaces);
+
+    cudaEvent_t events[2];
+    for (auto & event : events) {
+      cudaError_t result = cudaEventCreate(&event);
+      if (result != cudaSuccess) {
+        EXPECT_EQ(result, cudaSuccess) << " cudaEventCreate() failed with error " << cudaGetErrorString(result);
+        return false;
+        break;
+      }
+    }
+
+    int const kWarmupIterations = 5;
+    int const kProfilingIterations = 100;
+
+    for (int i = 0; i < kWarmupIterations; ++i) {
+      status = gemm_op();
+      EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
+    }
+    
+
+    cudaError_t result = cudaEventRecord(events[0]);
+    EXPECT_EQ(result, cudaSuccess);
+
+    for (int i = 0; i < kProfilingIterations; ++i) {
+
+      typename Gemm::Arguments arguments{
+        mode,
+        problem_size,
+        batch_count,
+        {alpha, beta},
+        profiling_tensor_A.get() + tensor_A.capacity() * (i % kWorkspaces),
+        profiling_tensor_B.get() + tensor_B.capacity() * (i % kWorkspaces),
+        profiling_tensor_C.get() + tensor_C.capacity() * (i % kWorkspaces),
+        profiling_tensor_Z.get() + tensor_Z.capacity() * (i % kWorkspaces),
+        profiling_tensor_Broadcast.get() + tensor_Broadcast.capacity() * (i % kWorkspaces),
+        profiling_tensor_T.get() + tensor_T.capacity() * (i % kWorkspaces),
+        problem_size.m() * problem_size.k(),
+        problem_size.n() * problem_size.k(),
+        problem_size.m() * problem_size.n(),
+        problem_size.m() * problem_size.n(),
+        problem_size.m(),
+        problem_size.m() * problem_size.n(),
+        tensor_A.layout().stride(0),
+        tensor_B.layout().stride(0),
+        tensor_C.layout().stride(0),
+        tensor_Z.layout().stride(0),
+        0,                                    // This must be zero
+        tensor_T.layout().stride(0),
+      };
+
+      gemm_op.initialize(arguments, workspace.get());
+      status = gemm_op();
+      EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
+    }
+
+    result = cudaEventRecord(events[1]);
+    EXPECT_EQ(result, cudaSuccess);
+
+    result = cudaDeviceSynchronize();
+    EXPECT_EQ(result, cudaSuccess);
+
+    float elapsed_time = 0;
+    result = cudaEventElapsedTime(&elapsed_time, events[0], events[1]);
+    EXPECT_EQ(result, cudaSuccess);
+
+    double average_time = double(elapsed_time) / double(kProfilingIterations);
+
+    std::cout << problem_size << ": " << average_time << " ms" << std::endl;
+
+    for (auto & event : events) {
+      cudaEventDestroy(event);
+    }
+    #endif
+
+    return passed;
+  }
+};
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <
+  typename Gemm, 
+  typename ReferenceOp = GemmWithBroadcastReferenceOp<Gemm>
+>
+bool TestGemmWithBroadcast(
+  cutlass::gemm::GemmCoord const & problem_size,
+  cutlass::gemm::GemmUniversalMode mode,
+  int batch_count,
+  double alpha = 1.0, 
+  double beta = 2.0) {
+
+  bool passed = true;
+
+  TestbedGemmWithBroadcast<Gemm, ReferenceOp> testbed;
+  
+  using ElementAccumulator = typename Gemm::ElementAccumulator;
+
+  passed = testbed.run(
+    mode,
+    problem_size, 
+    batch_count,
+    cutlass::from_real<ElementAccumulator>(alpha), 
+    cutlass::from_real<ElementAccumulator>(beta)
+  );
+
+  return passed;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <
+  typename Gemm, 
+  typename ReferenceOp = GemmWithBroadcastReferenceOp<Gemm>
+>
+bool TestAllGemmWithBroadcast() {
+
+  int M_problems[] = {8, 136, 264, 520};
+  int N_problems[] = {8, 136, 264, 520};
+  int K_problems[] = {8, 136, 264, 520};
+  double alpha_problems[] = {1.25, 2.25};
+  double beta_problems[] = {0, 1, 2.0};
+
+  bool passed = true;
+
+  for (int M : M_problems) {
+    for (int N : N_problems) {
+      for (int K : K_problems) {
+        for (double alpha : alpha_problems) {
+          for (double beta : beta_problems) {
+
+            TestbedGemmWithBroadcast<Gemm, ReferenceOp> testbed;
+            
+            using ElementAccumulator = typename Gemm::ElementAccumulator;
+
+            passed = testbed.run(
+              cutlass::gemm::GemmUniversalMode::kGemm,
+              {M, N, K}, 
+              1,
+              cutlass::from_real<ElementAccumulator>(alpha), 
+              cutlass::from_real<ElementAccumulator>(beta)
+            );
+            
+            EXPECT_TRUE(passed) 
+              << "M: " << M << ", N: " << N << ", K: " << K << ", alpha: " << alpha << ", beta: " << beta;
+
+            if (!passed) {
+
+              return passed;
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return passed;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace device
+} // namespace gemm
+} // namespace test
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+

test/unit/gemm/device/testbed_gemm_with_reduction.h

@@ -0,0 +1,491 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+/*! \file
+    \brief Tests for device-wide GEMM interface
+*/
+
+#pragma once
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+
+#include "../../common/cutlass_unit_test.h"
+
+#include "cutlass/util/host_tensor.h"
+#include "cutlass/util/tensor_view_io.h"
+#include "cutlass/util/distribution.h"
+#include "cutlass/util/reference/host/tensor_fill.h"
+#include "cutlass/util/reference/host/tensor_copy.h"
+#include "cutlass/util/reference/host/tensor_compare.h"
+#include "cutlass/util/reference/host/tensor_norm.h"
+#include "cutlass/util/reference/host/gemm.h"
+#include "cutlass/util/reference/host/gemm_complex.h"
+
+#include "testbed_utils.h"
+
+namespace test {
+namespace gemm {
+namespace device {
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename Gemm, typename BinaryOp>
+struct GemmWithReductionReference {
+  using ElementAccumulator = typename Gemm::ElementAccumulator;
+  using ElementCompute = typename Gemm::GemmKernel::Epilogue::ElementCompute;
+  using ElementC = typename Gemm::ElementC;
+  using ElementT = typename Gemm::GemmKernel::Epilogue::ElementTensor;
+  //
+  // Data members
+  //
+
+  BinaryOp binary_op;
+
+  //
+  // Methods
+  //
+
+  GemmWithReductionReference() { }
+
+  ElementCompute operator()(
+    ElementAccumulator d_y, 
+    ElementT t) {
+    
+    return binary_op(ElementCompute(d_y), ElementCompute(t));
+  }
+};
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <
+  typename Gemm,
+  typename ReferenceOp
+>
+struct TestbedGemmWithReduction {
+
+  using ElementAccumulator = typename Gemm::ElementAccumulator;
+  using ElementT = typename Gemm::GemmKernel::Epilogue::ElementTensor;
+
+  /// Initialization
+  cutlass::Distribution::Kind init_A;
+  cutlass::Distribution::Kind init_B;
+  cutlass::Distribution::Kind init_C;
+  uint64_t seed;
+
+  cutlass::HostTensor<typename Gemm::ElementA, typename Gemm::LayoutA> tensor_A;
+  cutlass::HostTensor<typename Gemm::ElementB, typename Gemm::LayoutB> tensor_B;
+  cutlass::HostTensor<typename Gemm::ElementC, typename Gemm::LayoutC> tensor_C;
+  cutlass::HostTensor<typename Gemm::ElementC, typename Gemm::LayoutC> tensor_D;
+  cutlass::HostTensor<typename Gemm::ElementAccumulator, typename Gemm::LayoutC> tensor_Reduction;
+  cutlass::HostTensor<ElementT, typename Gemm::LayoutC> tensor_Tensor;
+  cutlass::HostTensor<ElementAccumulator, typename Gemm::LayoutC> tensor_C_ref;
+  cutlass::HostTensor<ElementAccumulator, typename Gemm::LayoutC> reference_d_Y;
+  cutlass::HostTensor<typename Gemm::ElementC, typename Gemm::LayoutC> reference_D;
+  cutlass::HostTensor<typename Gemm::ElementAccumulator, typename Gemm::LayoutC> reference_Reduction;
+
+  //
+  // Methods
+  //
+
+  TestbedGemmWithReduction(
+    cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
+    cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
+    cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
+    uint64_t seed_ = 2080
+  ):
+    init_A(init_A_), init_B(init_B_), init_C(init_C_), seed(seed_) { }
+
+  /// Helper to initialize a tensor view
+  template <typename Element, typename Layout>
+  bool initialize_tensor(
+    cutlass::TensorView<Element, Layout> view, 
+    cutlass::Distribution::Kind dist_kind,
+    uint64_t seed) {
+
+    if (dist_kind == cutlass::Distribution::Uniform) {
+
+      double scope_max, scope_min;
+      int bits_input = cutlass::sizeof_bits<Element>::value;
+      int bits_output = cutlass::sizeof_bits<typename Gemm::ElementC>::value;
+
+      if (bits_input == 1) {
+        scope_max = 2;
+        scope_min = 0;
+      } else if (bits_input <= 8) {
+        scope_max = 2;
+        scope_min = -2;
+      } else if (bits_output == 16) {
+        scope_max = 5;
+        scope_min = -5;
+      } else {
+        scope_max = 8;
+        scope_min = -8;
+      }
+
+      cutlass::reference::host::TensorFillRandomUniform(
+        view, seed, scope_max, scope_min, 0);
+    } 
+    else if (dist_kind == cutlass::Distribution::Identity) {
+
+      cutlass::reference::host::TensorFillIdentity(view);
+    } 
+    else if (dist_kind == cutlass::Distribution::Gaussian) {
+
+      cutlass::reference::host::TensorFillRandomGaussian(view, seed, 0, 0.5);
+    }
+    else if (dist_kind == cutlass::Distribution::Sequential) {
+
+      for (int m = 0; m < view.extent().row(); ++m) {
+        for (int n = 0; n < view.extent().column(); ++n) {
+          //view.at({m, n}) = Element(float(((idx ++) % 17) - 8));
+          view.at({m, n}) = (n == 0 ? Element(m) : Element());
+
+        }
+      }
+    } 
+    else {
+      // TODO: Implement the rest
+      EXPECT_TRUE(false) << "Not implemented";
+      return false;
+    }
+
+    return true;
+  }
+
+  /// Initializes data structures
+  void initialize(cutlass::gemm::GemmCoord problem_size) {
+    //
+    // Allocate the GEMM workspace
+    //
+
+    tensor_A.resize(problem_size.mk());
+    tensor_B.resize(problem_size.kn());
+    tensor_C.resize(problem_size.mn());
+    tensor_D.resize(problem_size.mn());
+
+    tensor_Reduction.resize({
+      problem_size.m(), 
+      (problem_size.n() - 1 + Gemm::ThreadblockShape::kN) / Gemm::ThreadblockShape::kN
+    });
+
+    tensor_Tensor.resize(problem_size.mn());
+    reference_D.resize(problem_size.mn(), false);
+    reference_d_Y.resize(problem_size.mn(), false);
+    tensor_C_ref.resize(problem_size.mn(), false);
+    reference_Reduction.resize({problem_size.m(), 1}, false);
+
+    EXPECT_TRUE(initialize_tensor(tensor_A.host_view(), init_A, seed + 2019));
+    EXPECT_TRUE(initialize_tensor(tensor_B.host_view(), init_B, seed + 2018));
+    EXPECT_TRUE(initialize_tensor(tensor_C.host_view(), init_C, seed + 2017));
+    EXPECT_TRUE(initialize_tensor(tensor_Tensor.host_view(), init_C, seed + 2020));
+
+    // It is possible to randomly initialize to all zeros, so override this with non-zeros
+    // in the upper left corner of each operand.
+    tensor_A.host_view().at({0, 0}) = typename Gemm::ElementA(1);
+    tensor_B.host_view().at({0, 0}) = typename Gemm::ElementB(1);
+    tensor_C.host_view().at({0, 0}) = typename Gemm::ElementC(1);
+
+    for (int m = 0; m < tensor_C_ref.extent().row(); ++m) {
+      for (int n = 0; n < tensor_C_ref.extent().column(); ++n) {
+        tensor_C_ref.at({m, n}) = ElementAccumulator(tensor_C.at({m, n}));
+      }
+    }
+
+    tensor_A.sync_device();
+    tensor_B.sync_device();
+    tensor_C.sync_device();
+    tensor_D.sync_device();
+    tensor_Reduction.sync_device();
+    tensor_Tensor.sync_device();
+  }
+
+  /// Compares computed reference with device reference and outputs to a file if incorrect
+  bool compare_reference(
+    cutlass::gemm::GemmCoord problem_size, 
+    ElementAccumulator alpha, 
+    ElementAccumulator beta) {
+
+    tensor_Reduction.sync_host();
+    tensor_D.sync_host();
+
+    EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_A.host_view()), 0);
+    EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_B.host_view()), 0);
+    EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_C.host_view()), 0);
+    
+    EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_D.host_view()), 0);
+    EXPECT_GT(cutlass::reference::host::TensorNorm(reference_D.host_view()), 0);
+    EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_Reduction.host_view()), 0);
+
+    bool passed = true;
+    for (int m = 0; m < tensor_Reduction.extent().row(); ++m) {
+
+      ElementAccumulator reduced_value = ElementAccumulator();
+      for (int j = 0; j < tensor_Reduction.extent().column(); ++j) {
+        reduced_value += tensor_Reduction.at({m, j});
+      }
+
+      if (reduced_value != reference_Reduction.at({m, 0})) {
+        std::cout << "Error in bias[" << m << "] - Expected: " << reference_Reduction.at({m, 0}) << ", got: " << reduced_value << std::endl;
+        passed = false;
+        break;
+      }
+    }
+    EXPECT_TRUE(passed) << "Reduction is incorect.";
+
+    if (!cutlass::reference::host::TensorEquals(reference_D.host_view(), tensor_D.host_view())) {
+      EXPECT_TRUE(false) << " mismatched reference";
+      passed = false;
+    }
+    
+    if (!passed) {
+
+      /*
+      std::stringstream fname;
+
+      fname << "error_Gemm_device_"
+        << problem_size.m() << "x"
+        << problem_size.n() << "x"
+        << problem_size.k() << "_"
+        << Gemm::ThreadblockShape::kM << "x"  
+        << Gemm::ThreadblockShape::kN << "x"  
+        << Gemm::ThreadblockShape::kK << "_"
+        << Gemm::WarpShape::kM << "x"  
+        << Gemm::WarpShape::kN << "x"  
+        << Gemm::WarpShape::kK << ".txt";
+
+      std::ofstream file(fname.str());
+      */
+
+      std::ofstream file("testbed_universal_errors_sm70.txt");
+
+      file
+        << "problem: " << problem_size 
+        << ", alpha: " << alpha << ", beta: " << beta << "\n\n";
+
+      file 
+        << "A =\n" << tensor_A.host_view()
+        << "\nB =\n" << tensor_B.host_view()
+        << "\nC =\n" << tensor_C.host_view()
+        << "\nT = \n" << tensor_Tensor.host_view()
+        << "\n\nReference =\n" << reference_D.host_view()
+        << "\nComputed =\n" << tensor_D.host_view()
+        << "\n\nReduction =\n" << tensor_Reduction.host_view() << "\n"
+        << "\nReference reduction =\n" << reference_Reduction.host_view() << "\n";
+    }
+
+    return passed;
+  }
+
+  /// Verifies the result is a GEMM
+  bool verify(
+    cutlass::gemm::GemmCoord problem_size, 
+    ElementAccumulator alpha, 
+    ElementAccumulator beta) {
+
+    //
+    // Verify
+    //
+
+    cutlass::reference::host::GemmComplex<
+        typename Gemm::ElementA, typename Gemm::LayoutA,
+        typename Gemm::ElementB, typename Gemm::LayoutB,
+        ElementAccumulator, typename Gemm::LayoutC, 
+        ElementAccumulator, ElementAccumulator
+    >(
+      problem_size,
+      alpha, 
+      tensor_A.host_ref(),
+      Gemm::kTransformA,
+      tensor_B.host_ref(),
+      Gemm::kTransformB,
+      beta, 
+      tensor_C_ref.host_ref(), 
+      reference_d_Y.host_ref(), 
+      ElementAccumulator(0)
+    );
+
+    using ElementC = typename Gemm::ElementC;
+
+    ReferenceOp reference_op;
+
+    // compute backwards 
+    for (int m = 0; m < problem_size.m(); ++m) {
+      ElementAccumulator reduced_value = ElementAccumulator();
+      for (int n = 0; n < problem_size.n(); ++n) {
+        ElementAccumulator d_full = reference_op(reference_d_Y.at({m, n}), tensor_Tensor.at({m, n}));
+        reduced_value += d_full;
+        reference_D.at({m, n}) = ElementC(d_full);
+      }
+      reference_Reduction.at({m, 0}) = reduced_value;
+    }
+
+    return compare_reference(problem_size, alpha, beta);
+  }
+
+  /// Returns true if the CUDA device is sufficient to execute the kernel.
+  bool sufficient() const {
+
+    //
+    // Determine SMEM requirements and waive if not satisfied
+    //
+
+    int smem_size = int(sizeof(typename Gemm::GemmKernel::SharedStorage));
+
+    cudaDeviceProp properties;
+    int device_idx;
+    cudaError_t result = cudaGetDevice(&device_idx);
+
+    if (result != cudaSuccess) {
+      throw std::runtime_error("cudaGetDevice() API call failed.");
+    }
+
+    result = cudaGetDeviceProperties(&properties, device_idx);
+
+    if (result != cudaSuccess) {
+      throw std::runtime_error("cudaGetDeviceProperties() failed");
+    }
+
+    if (properties.sharedMemPerMultiprocessor < smem_size) {
+      return false;
+    }
+
+    return true;
+  }
+
+  /// Executes one test
+  bool run(
+    cutlass::gemm::GemmUniversalMode mode,
+    cutlass::gemm::GemmCoord problem_size, 
+    int batch_count = 1,
+    ElementAccumulator alpha = ElementAccumulator(1), 
+    ElementAccumulator beta = ElementAccumulator(0)) {
+
+    // Waive test if insufficient CUDA device
+    if (!sufficient()) {
+      if (CUTLASS_TEST_UNIT_ENABLE_WARNINGS) {
+        std::cerr << "Test waived due to insufficient CUDA device." << std::endl;
+      }
+      return true;
+    }
+
+    this->initialize(problem_size);
+
+    //
+    // Initialize the GEMM operator
+    //
+
+    typename Gemm::Arguments arguments{
+      mode,
+      problem_size,
+      batch_count,
+      {alpha, beta},
+      tensor_A.device_data(),
+      tensor_B.device_data(),
+      tensor_C.device_data(),
+      tensor_D.device_data(),
+      tensor_Reduction.device_data(),
+      tensor_Tensor.device_data(),
+      problem_size.m() * problem_size.k(),
+      problem_size.n() * problem_size.k(),
+      problem_size.m() * problem_size.n(),
+      problem_size.m() * problem_size.n(),
+      problem_size.m(),
+      problem_size.m() * problem_size.n(),
+      tensor_A.layout().stride(0),
+      tensor_B.layout().stride(0),
+      tensor_C.layout().stride(0),
+      tensor_D.layout().stride(0),
+      tensor_Reduction.layout().stride(0),
+      tensor_Tensor.layout().stride(0),
+    };
+
+    Gemm gemm_op;
+
+    size_t workspace_size = Gemm::get_workspace_size(arguments);
+
+    cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
+
+    cutlass::Status status = gemm_op.initialize(arguments, workspace.get());
+
+
+    EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
+
+    //
+    // Run the GEMM
+    //
+
+    status = gemm_op();
+
+    EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
+
+    //
+    // Verify
+    //
+
+    bool passed = this->verify(problem_size, alpha, beta);
+
+    if (!passed) {
+      std::cout << "Failed with batch_count/split_k_slices = " << batch_count << std::endl;
+    }
+
+    return passed;
+  }
+};
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+template <typename Gemm, typename ReferenceOp>
+bool TestGemmWithReduction(
+  cutlass::gemm::GemmCoord const & problem_size,
+  cutlass::gemm::GemmUniversalMode mode,
+  int batch_count = 1,
+  double alpha = 1.0, 
+  double beta = 2.0) {
+
+  bool passed = true;
+
+  TestbedGemmWithReduction<Gemm, ReferenceOp> testbed;
+  
+  using ElementAccumulator = typename Gemm::ElementAccumulator;
+
+  passed = testbed.run(
+    mode,
+    problem_size, 
+    batch_count,
+    cutlass::from_real<ElementAccumulator>(alpha), 
+    cutlass::from_real<ElementAccumulator>(beta)
+  );
+
+  return passed;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace device
+} // namespace gemm
+} // namespace test
+
+/////////////////////////////////////////////////////////////////////////////////////////////////

test/unit/gemm/device/testbed_planar_complex.h

@@ -103,8 +103,8 @@ public:
     cutlass::reference::host::TensorFillRandomUniform(
         tensor_C.host_view(), seed * 2020, scope_max, scope_min, 0);
 
-    cutlass::reference::host::TensorFill(tensor_D.host_view());
-    cutlass::reference::host::TensorFill(tensor_D_ref.host_view());
+    cutlass::reference::host::TensorFill(tensor_D.host_view(), cutlass::complex<ElementC>());
+    cutlass::reference::host::TensorFill(tensor_D_ref.host_view(), cutlass::complex<ElementC>());
 
     tensor_A.sync_device();
     tensor_B.sync_device();
@@ -162,10 +162,10 @@ public:
     ElementC *ptr_C = tensor_C.device_data();
     ElementC *ptr_D = tensor_D.device_data();
 
-    int lda = tensor_A.layout().stride(0);
-    int ldb = tensor_B.layout().stride(0);
-    int ldc = tensor_C.layout().stride(0);
-    int ldd = tensor_D.layout().stride(0);
+    typename LayoutA::Stride::Index lda = tensor_A.layout().stride(0);
+    typename LayoutB::Stride::Index ldb = tensor_B.layout().stride(0);
+    typename LayoutC::Stride::Index ldc = tensor_C.layout().stride(0);
+    typename LayoutC::Stride::Index ldd = tensor_D.layout().stride(0);
 
     int64_t imag_stride_A = tensor_A.imaginary_stride();
     int64_t imag_stride_B = tensor_B.imaginary_stride();
@@ -266,15 +266,15 @@ template <typename Gemm>
 bool TestAllGemmPlanarComplex() {
 
   int M[] = {
-    16, 264,
+    16, 64, 72, 144, 264, 520,
   };
 
   int N[] = {
-    16, 248,
+    16, 64, 72, 144, 248, 264, 520
   };
 
   int K[] = {
-    8, 96,
+    8, 64, 72, 96,  264, 520
   };
 
   using ElementCompute = typename Gemm::EpilogueOutputOp::ElementCompute;

test/unit/gemm/device/testbed_sparse.h

@@ -477,4 +477,3 @@ bool TestAllSparseGemm() {
 } // namespace test
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
-

test/unit/gemm/threadblock/mma_multistage.cu

@@ -22,6 +22,7 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  **************************************************************************************************/
+
 /*! \file
     \brief Unit tests for threadblock-level GEMM
 */
@@ -3824,4 +3825,5 @@ TEST(SM80_gemm_threadblock_crosswise_f64,
 }
 
 ////////////////////////////////////////////////////////////////////////////////
+
 #endif

test/unit/gemm/threadblock/mma_multistage_sparse_testbed.h

@@ -59,7 +59,8 @@ __global__ void kernel_multistage_mma_sparse(cutlass::gemm::GemmCoord problem_si
                                       typename Mma::IteratorA::TensorRef ref_A,
                                       typename Mma::IteratorB::Params params_B,
                                       typename Mma::IteratorB::TensorRef ref_B,
-                                      typename Mma::ElementC *ptr_C, int ldc,
+                                      typename Mma::ElementC *ptr_C,
+                                      typename Mma::LayoutC::Stride::Index ldc,
                                       typename Mma::IteratorE::Params params_E,
                                       typename Mma::IteratorE::TensorRef ref_E) {
   // Shared storage needed by threadblock-scoped matrix multiply-

test/unit/gemm/threadblock/mma_multistage_testbed.h

@@ -57,7 +57,8 @@ __global__ void kernel_multistage_mma(cutlass::gemm::GemmCoord problem_size,
                                       typename Mma::IteratorA::TensorRef ref_A,
                                       typename Mma::IteratorB::Params params_B,
                                       typename Mma::IteratorB::TensorRef ref_B,
-                                      typename Mma::ElementC *ptr_C, int ldc) {
+                                      typename Mma::ElementC *ptr_C, 
+                                      typename Mma::LayoutC::Stride::Index ldc) {
   // Shared storage needed by threadblock-scoped matrix multiply-accumulate
 
   // Dynamic shared memory base pointer

test/unit/gemm/threadblock/mma_pipelined_testbed.h

@@ -67,7 +67,8 @@ __global__ void kernel_mma(cutlass::gemm::GemmCoord problem_size,
                            typename Mma::IteratorA::TensorRef ref_A,
                            typename Mma::IteratorB::Params params_B,
                            typename Mma::IteratorB::TensorRef ref_B,
-                           typename Mma::ElementC *ptr_C, int ldc) {
+                           typename Mma::ElementC *ptr_C,
+                           typename Mma::LayoutC::Stride::Index ldc) {
   // Shared storage needed by threadblock-scoped matrix multiply-accumulate
   __shared__ typename Mma::SharedStorage shared_storage;
 

test/unit/gemm/threadblock/mma_planar_complex_testbed.h

@@ -67,7 +67,8 @@ __global__ void kernel_mma_planar_complex(
   typename Mma::IteratorB::Params params_B,
   typename Mma::IteratorB::Element *ptr_B,
   int64_t imaginary_stride_B,
-  typename Mma::ElementC *ptr_C, int ldc, int64_t imaginary_stride_C) {
+  typename Mma::ElementC *ptr_C, 
+  typename Mma::LayoutC::Stride::Index ldc, int64_t imaginary_stride_C) {
 
   // Shared storage needed by threadblock-scoped matrix multiply-accumulate
   __shared__ typename Mma::SharedStorage shared_storage;

test/unit/gemm/warp/gemm_sm50.cu

@@ -29,6 +29,7 @@
 #include "../../common/cutlass_unit_test.h"
 
 #include "cutlass/complex.h"
+#include "cutlass/quaternion.h"
 #include "cutlass/gemm/gemm.h"
 #include "cutlass/gemm/warp/mma_simt.h"
 
@@ -593,3 +594,55 @@ TEST(SM50_warp_gemm_complex_f64_col_row_row, 32x16x1_1x1x1) {
   test::gemm::warp::Testbed<Mma, cutlass::gemm::GemmShape<128, 128, 8>>().run();
 }
 /////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM50_warp_gemm_quaternion_f32_col_row_col, 16x8x8_1x1x1) {
+
+  using Policy = cutlass::gemm::warp::MmaSimtPolicy<
+    cutlass::MatrixShape<8, 4>,
+    cutlass::layout::ColumnMajorInterleaved<2>,
+    cutlass::gemm::GemmShape<1, 1, 1>
+  >;
+
+  using quaternion_f32_t = cutlass::Quaternion<float>;
+
+  using Mma = cutlass::gemm::warp::MmaSimt<
+    cutlass::gemm::GemmShape<16, 8, 8>,
+    quaternion_f32_t,
+    cutlass::layout::ColumnMajor,
+    quaternion_f32_t,
+    cutlass::layout::RowMajor,
+    quaternion_f32_t,
+    cutlass::layout::ColumnMajor,
+    Policy
+  >;
+
+  test::gemm::warp::Testbed<Mma, cutlass::gemm::GemmShape<128, 128, 8>>().run();
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM50_warp_gemm_quaternion_f32_col_row_row, 16x8x8_1x1x1) {
+
+  using Policy = cutlass::gemm::warp::MmaSimtPolicy<
+    cutlass::MatrixShape<8, 4>,
+    cutlass::layout::ColumnMajorInterleaved<2>,
+    cutlass::gemm::GemmShape<1, 1, 1>
+  >;
+
+  using quaternion_f32_t = cutlass::Quaternion<float>;
+
+  using Mma = cutlass::gemm::warp::MmaSimt<
+    cutlass::gemm::GemmShape<16, 8, 8>,
+    quaternion_f32_t,
+    cutlass::layout::ColumnMajor,
+    quaternion_f32_t,
+    cutlass::layout::RowMajor,
+    quaternion_f32_t,
+    cutlass::layout::RowMajor,
+    Policy
+  >;
+
+  test::gemm::warp::Testbed<Mma, cutlass::gemm::GemmShape<128, 128, 8>>().run();
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////

test/unit/gemm/warp/gemm_sm80.cu

@@ -1856,3 +1856,4 @@ TEST(SM80_warp_gemm_tensor_op_canonical_tf32_col_row, 32x32x8_64x32x8_8x8x4) {
 
 #endif // if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
 
+

test/unit/gemm/warp/testbed.h

@@ -33,6 +33,7 @@
 #include "cutlass/numeric_types.h"
 #include "cutlass/subbyte_reference.h"
 #include "cutlass/platform/platform.h"
+#include "cutlass/arch/arch.h"
 
 #include "cutlass/util/host_tensor.h"
 #include "cutlass/util/tensor_view_io.h"
@@ -100,9 +101,9 @@ __global__ void kernel(
   typename Mma::LayoutB layout_B = Mma::LayoutB::packed({ThreadblockShape::kK, ThreadblockShape::kN});
   typename Mma::LayoutC layout_C = Mma::LayoutC::packed({Mma::Shape::kM, Mma::Shape::kN});
 
-  typename Mma::IteratorA iter_A({smem_buffer_A.data(), layout_A}, cutlass::LaneId());
+  typename Mma::IteratorA iter_A({smem_buffer_A.data(), layout_A}, cutlass::arch::LaneId());
 
-  typename Mma::IteratorB iter_B({smem_buffer_B.data(), layout_B}, cutlass::LaneId());
+  typename Mma::IteratorB iter_B({smem_buffer_B.data(), layout_B}, cutlass::arch::LaneId());
 
   FragmentA frag_A;
   FragmentB frag_B;
@@ -129,7 +130,7 @@ __global__ void kernel(
     }
   }
   
-  typename Mma::IteratorC iter_C({output_C, layout_C}, cutlass::LaneId());
+  typename Mma::IteratorC iter_C({output_C, layout_C}, cutlass::arch::LaneId());
 
   iter_C.store(accum);
 }
@@ -142,7 +143,7 @@ template <
   typename Mma_,
   /// Size of threadblock-scoped shape used to store SMEM
   typename ThreadblockShape_,
-  /// The innter product operation performed by GEMM 
+  /// The inner product operation performed by GEMM 
   typename Operator_ = cutlass::arch::OpMultiplyAdd
 >
 struct Testbed {
@@ -205,8 +206,10 @@ struct Testbed {
       }
 
       uint64_t seed = 7;
-      cutlass::reference::host::TensorFillRandomUniform(
-          tensor_A.host_view(), seed, scope_max, scope_min, 0);
+
+      cutlass::reference::host::BlockFillRandomUniform(tensor_A.host_data(),
+        tensor_A.capacity(), seed, scope_max, scope_min, 0);
+
     } else if (init_A == cutlass::Distribution::Sequential) {
       cutlass::reference::host::BlockFillSequential(tensor_A.host_data(),
                                                     tensor_A.capacity());
@@ -230,8 +233,10 @@ struct Testbed {
       }
 
       uint64_t seed = 7;
-      cutlass::reference::host::TensorFillRandomUniform(
-          tensor_B.host_view(), seed + 16, scope_max, scope_min, 0);
+
+      cutlass::reference::host::BlockFillRandomUniform(tensor_B.host_data(),
+        tensor_B.capacity(), seed, scope_max, scope_min, 0);
+
     } else if (init_B == cutlass::Distribution::Sequential) {
       cutlass::reference::host::BlockFillSequential(tensor_B.host_data(),
                                                     tensor_B.capacity());
@@ -313,23 +318,25 @@ struct Testbed {
 
       cutlass::TensorView<ElementA, cutlass::layout::ColumnMajor> tensor_A_physical(
         tensor_A.host_data(), 
-        tensor_A.stride(), 
+        tensor_A.stride()[0], 
         tensor_A.extent());
 
       cutlass::TensorView<ElementB, cutlass::layout::RowMajor> tensor_B_physical(
         tensor_B.host_data(), 
-        tensor_B.stride(), 
+        tensor_B.stride()[0], 
         tensor_B.extent());
 
       std::cout <<"cutlass::sizeof_bits<ElementA>::value = "<<cutlass::sizeof_bits<ElementA>::value<<"\n";
       std::cout
         << "A:\n" << tensor_A.host_view() << "\n\n"
-        << "A(physical - stride: " << tensor_A.stride() << ", extent: " << tensor_A.extent() << "):\n" << tensor_A_physical << "\n\n";
+        << "A(physical - stride: " << tensor_A.stride()[0] 
+        << ", extent: " << tensor_A.extent() << "):\n" << tensor_A_physical << "\n\n";
 
       std::cout <<"cutlass::sizeof_bits<ElementB>::value = "<<cutlass::sizeof_bits<ElementB>::value<<"\n";
       std::cout
         << "B:\n" << tensor_B.host_view() << "\n\n"
-        << "B(physical - stride: " << tensor_B.stride() << ", extent: " << tensor_B.extent() << "):\n" << tensor_B_physical << "\n\n";
+        << "B(physical - stride: " << tensor_B.stride()[0] 
+        << ", extent: " << tensor_B.extent() << "):\n" << tensor_B_physical << "\n\n";
 
       std::cout
         << "C:\n" << tensor_C.host_view() << "\n\n"
@@ -493,23 +500,23 @@ struct TestbedComplex {
 
       cutlass::TensorView<ElementA, cutlass::layout::ColumnMajor> tensor_A_physical(
         tensor_A.host_data(), 
-        tensor_A.stride(), 
+        tensor_A.stride()[0], 
         tensor_A.extent());
 
       cutlass::TensorView<ElementB, cutlass::layout::RowMajor> tensor_B_physical(
         tensor_B.host_data(), 
-        tensor_B.stride(), 
+        tensor_B.stride()[0], 
         tensor_B.extent());
 
       std::cout <<"cutlass::sizeof_bits<ElementA>::value = "<<cutlass::sizeof_bits<ElementA>::value<<"\n";
       std::cout 
         << "A:\n" << tensor_A.host_view() << "\n\n"
-        << "A(physical - stride: " << tensor_A.stride() << ", extent: " << tensor_A.extent() << "):\n" << tensor_A_physical << "\n\n";
+        << "A(physical - stride: " << tensor_A.stride()[0] << ", extent: " << tensor_A.extent() << "):\n" << tensor_A_physical << "\n\n";
 
       std::cout <<"cutlass::sizeof_bits<ElementB>::value = "<<cutlass::sizeof_bits<ElementB>::value<<"\n";
       std::cout
         << "B:\n" << tensor_B.host_view() << "\n\n"
-        << "B(physical - stride: " << tensor_B.stride() << ", extent: " << tensor_B.extent() <<"):\n" << tensor_B_physical << "\n\n";
+        << "B(physical - stride: " << tensor_B.stride()[0] << ", extent: " << tensor_B.extent() <<"):\n" << tensor_B_physical << "\n\n";
 
       std::cout
         << "C:\n" << tensor_C.host_view() << "\n\n"
@@ -574,9 +581,9 @@ __global__ void kernel_transform(
   typename Mma::LayoutB layout_B = Mma::LayoutB::packed({ThreadblockShape::kK, ThreadblockShape::kN});
   typename Mma::LayoutC layout_C = Mma::LayoutC::packed({Mma::Shape::kM, Mma::Shape::kN});
 
-  typename Mma::IteratorA iter_A({smem_buffer_A.data(), layout_A}, cutlass::LaneId());
+  typename Mma::IteratorA iter_A({smem_buffer_A.data(), layout_A}, cutlass::arch::LaneId());
 
-  typename Mma::IteratorB iter_B({smem_buffer_B.data(), layout_B}, cutlass::LaneId());
+  typename Mma::IteratorB iter_B({smem_buffer_B.data(), layout_B}, cutlass::arch::LaneId());
 
   FragmentA loaded_frag_A;
   FragmentB loaded_frag_B;
@@ -608,7 +615,7 @@ __global__ void kernel_transform(
     }
   }
   
-  typename Mma::IteratorC iter_C({output_C, layout_C}, cutlass::LaneId());
+  typename Mma::IteratorC iter_C({output_C, layout_C}, cutlass::arch::LaneId());
 
   iter_C.store(accum);
 }
@@ -790,23 +797,23 @@ struct TransformTestbed {
 
       cutlass::TensorView<ElementA, cutlass::layout::ColumnMajor> tensor_A_physical(
         tensor_A.host_data(), 
-        tensor_A.stride(), 
+        tensor_A.stride()[0], 
         tensor_A.extent());
 
       cutlass::TensorView<ElementB, cutlass::layout::RowMajor> tensor_B_physical(
         tensor_B.host_data(), 
-        tensor_B.stride(), 
+        tensor_B.stride()[0], 
         tensor_B.extent());
 
       std::cout <<"cutlass::sizeof_bits<ElementA>::value = "<<cutlass::sizeof_bits<ElementA>::value<<"\n";
       std::cout
         << "A:\n" << tensor_A.host_view() << "\n\n"
-        << "A(physical - stride: " << tensor_A.stride() << ", extent: " << tensor_A.extent() << "):\n" << tensor_A_physical << "\n\n";
+        << "A(physical - stride: " << tensor_A.stride()[0] << ", extent: " << tensor_A.extent() << "):\n" << tensor_A_physical << "\n\n";
 
       std::cout <<"cutlass::sizeof_bits<ElementB>::value = "<<cutlass::sizeof_bits<ElementB>::value<<"\n";
       std::cout
         << "B:\n" << tensor_B.host_view() << "\n\n"
-        << "B(physical - stride: " << tensor_B.stride() << ", extent: " << tensor_B.extent() << "):\n" << tensor_B_physical << "\n\n";
+        << "B(physical - stride: " << tensor_B.stride()[0] << ", extent: " << tensor_B.extent() << "):\n" << tensor_B_physical << "\n\n";
 
       std::cout
         << "C:\n" << tensor_C.host_view() << "\n\n"
@@ -970,23 +977,23 @@ struct TransformedTestbedComplex {
 
       cutlass::TensorView<ElementA, cutlass::layout::ColumnMajor> tensor_A_physical(
         tensor_A.host_data(), 
-        tensor_A.stride(), 
+        tensor_A.stride()[0], 
         tensor_A.extent());
 
       cutlass::TensorView<ElementB, cutlass::layout::RowMajor> tensor_B_physical(
         tensor_B.host_data(), 
-        tensor_B.stride(), 
+        tensor_B.stride()[0], 
         tensor_B.extent());
 
       std::cout <<"cutlass::sizeof_bits<ElementA>::value = "<<cutlass::sizeof_bits<ElementA>::value<<"\n";
       std::cout 
         << "A:\n" << tensor_A.host_view() << "\n\n"
-        << "A(physical - stride: " << tensor_A.stride() << ", extent: " << tensor_A.extent() << "):\n" << tensor_A_physical << "\n\n";
+        << "A(physical - stride: " << tensor_A.stride()[0] << ", extent: " << tensor_A.extent() << "):\n" << tensor_A_physical << "\n\n";
 
       std::cout <<"cutlass::sizeof_bits<ElementB>::value = "<<cutlass::sizeof_bits<ElementB>::value<<"\n";
       std::cout
         << "B:\n" << tensor_B.host_view() << "\n\n"
-        << "B(physical - stride: " << tensor_B.stride() << ", extent: " << tensor_B.extent() <<"):\n" << tensor_B_physical << "\n\n";
+        << "B(physical - stride: " << tensor_B.stride()[0] << ", extent: " << tensor_B.extent() <<"):\n" << tensor_B_physical << "\n\n";
 
       std::cout
         << "C:\n" << tensor_C.host_view() << "\n\n"
@@ -1073,11 +1080,11 @@ __global__ void sparse_kernel(
                             Mma::Shape::kK / Mma::kSparse /
                                 Mma::kElementsPerElementE / Mma::kInterleaved});
 
-  typename Mma::IteratorA iter_A({smem_buffer_A.data(), layout_A}, cutlass::LaneId());
+  typename Mma::IteratorA iter_A({smem_buffer_A.data(), layout_A}, cutlass::arch::LaneId());
 
-  typename Mma::IteratorB iter_B({smem_buffer_B.data(), layout_B}, cutlass::LaneId());
+  typename Mma::IteratorB iter_B({smem_buffer_B.data(), layout_B}, cutlass::arch::LaneId());
 
-  typename Mma::IteratorE iter_E({smem_buffer_E.data(), layout_E}, cutlass::LaneId());
+  typename Mma::IteratorE iter_E({smem_buffer_E.data(), layout_E}, cutlass::arch::LaneId());
 
   FragmentA frag_A;
   FragmentB frag_B;
@@ -1108,7 +1115,7 @@ __global__ void sparse_kernel(
     }
   }
   
-  typename Mma::IteratorC iter_C({output_C, layout_C}, cutlass::LaneId());
+  typename Mma::IteratorC iter_C({output_C, layout_C}, cutlass::arch::LaneId());
 
   iter_C.store(accum);
 }