CUTLASS 2.4 (Implicit GEMM convolution) (#147)

CUTLASS 2.4 (Implicit GEMM Convolution)

Co-authored-by: Manish Gupta <manigupta@nvidia.com>, Haicheng Wu <haichengw@nvidia.com>, Dustyn Blasig <dblasig@nvidia.com>, Andrew Kerr <akerr@nvidia.com>

test/unit/gemm/device/CMakeLists.txt

@@ -20,120 +20,47 @@
 # STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-cutlass_test_unit_add_executable(
+add_custom_target(
   cutlass_test_unit_gemm_device
+  DEPENDS
+  cutlass_test_unit_gemm_device_simt
+  cutlass_test_unit_gemm_device_tensorop_sm70
+  cutlass_test_unit_gemm_device_tensorop_sm75
+  cutlass_test_unit_gemm_device_tensorop_f16_sm80
+  cutlass_test_unit_gemm_device_tensorop_f32_sm80
+  cutlass_test_unit_gemm_device_tensorop_f32_tf32_sm80
+  cutlass_test_unit_gemm_device_tensorop_f64
+  cutlass_test_unit_gemm_device_tensorop_s32_sm80
+  cutlass_test_unit_gemm_device_wmma
+  cutlass_test_unit_gemm_device_tensorop_planar_complex
+  cutlass_test_unit_gemm_device_sparse_tensorop_sm80
+)
+
+add_custom_target(
+  test_unit_gemm_device
+  DEPENDS
+  test_unit_gemm_device_simt
+  test_unit_gemm_device_tensorop_sm70
+  test_unit_gemm_device_tensorop_sm75
+  test_unit_gemm_device_tensorop_f16_sm80
+  test_unit_gemm_device_tensorop_f32_sm80
+  test_unit_gemm_device_tensorop_f32_tf32_sm80
+  test_unit_gemm_device_tensorop_f64
+  test_unit_gemm_device_tensorop_s32_sm80
+  test_unit_gemm_device_wmma
+  test_unit_gemm_device_tensorop_planar_complex
+  test_unit_gemm_device_sparse_tensorop_sm80
+)
+
+cutlass_test_unit_add_executable(
+  cutlass_test_unit_gemm_device_simt
 
   BATCH_SOURCES ON
   BATCH_SIZE 4
 
-  gemm_planar_complex_f16_f16_f32_tensor_op_sm70.cu                              
-  gemm_planar_complex_f16_f16_f32_tensor_op_sm75.cu                                         
-  gemm_planar_complex_f16_f16_f32_tensor_op_sm80.cu
-
-  gemm_universal_f16n_f16t_f32t_tensor_op_f32_sm80.cu                                     
-  gemm_universal_cf64n_cf64t_cf64t_tensor_op_f64_sm80.cu                             
-  gemm_universal_cf64n_cf64t_cf64t_tensor_op_f64_gaussian_sm80.cu
-  gemm_universal_cf32n_cf32n_cf32n_tensor_op_f32_sm80.cu
-
-  gemm_cf64n_cf64t_cf64t_tensor_op_f64_sm80.cu
-  gemm_cf64t_cf64n_cf64t_tensor_op_f64_sm80.cu
-
-  gemm_cf64n_cf64t_cf64t_tensor_op_f64_gaussian_sm80.cu
-  gemm_cf64t_cf64n_cf64t_tensor_op_f64_gaussian_sm80.cu
-
-  gemm_cf32n_cf32t_cf32t_tensor_op_tf32_f32_sm80.cu
-  gemm_cf32t_cf32n_cf32t_tensor_op_tf32_f32_sm80.cu
-
-  gemm_f16n_f16n_f16t_tensor_op_f32_sm80.cu
-  gemm_f16n_f16n_f32n_tensor_op_f32_sm80.cu
-  gemm_f16n_f16n_f32t_tensor_op_f32_sm80.cu
-  gemm_f16n_f16t_f16t_tensor_op_f16_sm80.cu
-  gemm_f16n_f16t_f32t_tensor_op_f32_sm80.cu
-  gemm_f16t_f16n_f16t_tensor_op_f16_sm80.cu
-  gemm_f16t_f16n_f32t_tensor_op_f32_sm80.cu
-  gemm_f16t_f16t_f32n_tensor_op_f32_sm80.cu
-  gemm_f16t_f16t_f32t_tensor_op_f32_sm80.cu
-  gemm_bf16n_bf16n_f32t_tensor_op_f32_sm80.cu
-  gemm_bf16t_bf16t_bf16t_tensor_op_f32_sm80.cu
-  gemm_tf32t_tf32n_f32t_tensor_op_f32_sm80.cu
-  gemm_tf32n_tf32t_f32t_tensor_op_f32_sm80.cu
-  gemm_tf32n_tf32n_f32t_tensor_op_f32_sm80.cu
-  gemm_tf32t_tf32t_f32t_tensor_op_f32_sm80.cu
-
-  gemm_f16n_f16n_f16t_tensor_op_f32_sparse_sm80.cu
-  gemm_f16n_f16n_f32t_tensor_op_f32_sparse_sm80.cu
-  gemm_f16n_f16t_f16t_tensor_op_f16_sparse_sm80.cu
-  gemm_f16n_f16t_f32t_tensor_op_f32_sparse_sm80.cu
-  gemm_f16t_f16n_f16t_tensor_op_f16_sparse_sm80.cu
-  gemm_f16t_f16n_f32t_tensor_op_f32_sparse_sm80.cu
-  gemm_f16t_f16t_f32t_tensor_op_f32_sparse_sm80.cu
-  gemm_f32t_f32n_f32t_tensor_op_f32_sparse_sm80.cu
-  gemm_f32n_f32t_f32t_tensor_op_f32_sparse_sm80.cu
-  gemm_f32t_f32t_f32t_tensor_op_f32_sparse_sm80.cu
-  gemm_f32n_f32n_f32t_tensor_op_f32_sparse_sm80.cu
-  gemm_s8t_s8n_s32t_tensor_op_s32_sparse_sm80.cu
-  gemm_s4t_s4n_s32t_tensor_op_s32_sparse_sm80.cu
-
-  gemm_f16t_f16n_f16t_tensor_op_f16_slicedk_sm80.cu
-  gemm_f16n_f16t_f16t_tensor_op_f16_slicedk_sm80.cu
-
   simt_sgemm_nt_sm80.cu
   simt_sgemm_tn_sm80.cu
-
-  gemm_s8t_s8n_s32t_tensor_op_s32_sm80.cu
-  gemm_s8t_s8n_s32n_tensor_op_s32_sm80.cu
-  gemm_s8t_s8n_s8n_tensor_op_s32_sm80.cu
-  gemm_s8t_s8n_s8t_tensor_op_s32_sm80.cu
-  gemm_s4t_s4n_s32n_tensor_op_s32_sm80.cu
-  gemm_s4t_s4n_s32t_tensor_op_s32_sm80.cu
-  gemm_b1t_b1n_s32n_tensor_op_s32_sm80.cu
-  gemm_b1t_b1n_s32t_tensor_op_s32_sm80.cu
-
-  gemm_s8n_s8t_s8n_tensor_op_s32_sm80.cu
-  gemm_s4n_s4t_s4n_tensor_op_s32_sm80.cu
-
-  gemm_f64n_f64t_f64t_tensor_op_f64_sm80.cu
-  gemm_f64t_f64n_f64t_tensor_op_f64_sm80.cu
-
-  gemm_b1t_b1n_s32t_tensor_op_s32_sm75.cu
-  gemm_b1t_b1n_s32n_tensor_op_s32_sm75.cu
-
-  gemm_f32n_f32n_f32t_tensor_op_f32_sm80.cu
-  gemm_f16t_f16n_f16t_tensor_op_f16_sm75.cu
-  gemm_f16n_f16t_f16t_tensor_op_f16_sm75.cu
-  gemm_f16n_f16t_f16t_tensor_op_f16_slicedk_sm75.cu
-  gemm_f16t_f16n_f16t_tensor_op_f16_slicedk_sm75.cu
-
-  gemm_f16n_f16n_f16t_tensor_op_f32_sm75.cu
-
-  gemm_f16n_f16n_f32t_tensor_op_f32_sm75.cu
-  gemm_f16n_f16t_f32t_tensor_op_f32_sm75.cu
-  gemm_f16t_f16n_f32t_tensor_op_f32_sm75.cu
-  gemm_f16t_f16t_f32t_tensor_op_f32_sm75.cu
-
-  gemm_f16n_f16n_f32n_tensor_op_f32_sm75.cu
-  gemm_f16t_f16t_f32n_tensor_op_f32_sm75.cu
-
-  gemm_s8n_s8t_s8n_tensor_op_s32_sm75.cu
-  gemm_s8t_s8n_s32t_tensor_op_s32_sm75.cu
-  gemm_s8t_s8n_s32n_tensor_op_s32_sm75.cu
-  gemm_s8t_s8n_s8t_tensor_op_s32_sm75.cu
-  gemm_s8t_s8n_s8n_tensor_op_s32_sm75.cu
-
-  gemm_s4n_s4t_s4n_tensor_op_s32_sm75.cu
-  gemm_s4t_s4n_s32t_tensor_op_s32_sm75.cu
-  gemm_s4t_s4n_s32n_tensor_op_s32_sm75.cu
-
-  gemm_f16n_f16n_f32t_volta_tensor_op_f32_sm70.cu
-  gemm_f16n_f16t_f32t_volta_tensor_op_f32_sm70.cu
-  gemm_f16t_f16n_f32t_volta_tensor_op_f32_sm70.cu
-  gemm_f16t_f16t_f32t_volta_tensor_op_f32_sm70.cu
-
-  gemm_f16n_f16n_f16t_volta_tensor_op_f32_sm70.cu
-
-  gemm_f16n_f16t_f16t_volta_tensor_op_f16_sm70.cu
-  gemm_f16t_f16n_f16t_volta_tensor_op_f16_sm70.cu
-
+  
   simt_cgemm_nn_sm50.cu
   simt_cgemm_nt_sm50.cu
   simt_cgemm_tn_sm50.cu
@@ -167,10 +94,158 @@ cutlass_test_unit_add_executable(
   simt_zgemm_tn_sm50.cu
   simt_zgemm_tt_sm50.cu
 
+  gemm_splitk_simt_sm50.cu
+)
+
+cutlass_test_unit_add_executable(
+  cutlass_test_unit_gemm_device_tensorop_sm70
+
+  BATCH_SOURCES ON
+  BATCH_SIZE 4
+
+
+  gemm_f16n_f16n_f32t_volta_tensor_op_f32_sm70.cu
+  gemm_f16n_f16t_f32t_volta_tensor_op_f32_sm70.cu
+  gemm_f16t_f16n_f32t_volta_tensor_op_f32_sm70.cu
+  gemm_f16t_f16t_f32t_volta_tensor_op_f32_sm70.cu
+
+  gemm_f16n_f16n_f16t_volta_tensor_op_f32_sm70.cu
+
+  gemm_f16n_f16t_f16t_volta_tensor_op_f16_sm70.cu
+  gemm_f16t_f16n_f16t_volta_tensor_op_f16_sm70.cu
+
+  gemm_splitk_tensor_op_sm70.cu
+)
+
+cutlass_test_unit_add_executable(
+  cutlass_test_unit_gemm_device_tensorop_sm75
+
+  BATCH_SOURCES ON
+  BATCH_SIZE 4
+
+  gemm_f16t_f16n_f16t_tensor_op_f16_sm75.cu
+  gemm_f16n_f16t_f16t_tensor_op_f16_sm75.cu
+  gemm_f16n_f16t_f16t_tensor_op_f16_slicedk_sm75.cu
+  gemm_f16t_f16n_f16t_tensor_op_f16_slicedk_sm75.cu
+
+  gemm_f16n_f16n_f16t_tensor_op_f32_sm75.cu
+
+  gemm_f16n_f16n_f32t_tensor_op_f32_sm75.cu
+  gemm_f16n_f16t_f32t_tensor_op_f32_sm75.cu
+  gemm_f16t_f16n_f32t_tensor_op_f32_sm75.cu
+  gemm_f16t_f16t_f32t_tensor_op_f32_sm75.cu
+
+  gemm_f16n_f16n_f32n_tensor_op_f32_sm75.cu
+  gemm_f16t_f16t_f32n_tensor_op_f32_sm75.cu
+
+  gemm_s8n_s8t_s8n_tensor_op_s32_sm75.cu
+  gemm_s8t_s8n_s32t_tensor_op_s32_sm75.cu
+  gemm_s8t_s8n_s32n_tensor_op_s32_sm75.cu
+  gemm_s8t_s8n_s8t_tensor_op_s32_sm75.cu
+  gemm_s8t_s8n_s8n_tensor_op_s32_sm75.cu
+
+  gemm_s4n_s4t_s4n_tensor_op_s32_sm75.cu
+  gemm_s4t_s4n_s32t_tensor_op_s32_sm75.cu
+  gemm_s4t_s4n_s32n_tensor_op_s32_sm75.cu
+  
+  gemm_b1t_b1n_s32t_tensor_op_s32_sm75.cu
+  gemm_b1t_b1n_s32n_tensor_op_s32_sm75.cu
+
   gemm_splitk_serial_tensor_op_sm75.cu
   gemm_splitk_tensor_op_sm75.cu
-  gemm_splitk_tensor_op_sm70.cu
-  gemm_splitk_simt_sm50.cu
+
+)
+
+cutlass_test_unit_add_executable(
+  cutlass_test_unit_gemm_device_tensorop_f16_sm80
+
+  BATCH_SOURCES ON
+  BATCH_SIZE 4
+
+  gemm_f16t_f16n_f16t_tensor_op_f16_slicedk_sm80.cu
+  gemm_f16n_f16t_f16t_tensor_op_f16_slicedk_sm80.cu
+)
+
+cutlass_test_unit_add_executable(
+  cutlass_test_unit_gemm_device_tensorop_f32_sm80
+
+  BATCH_SOURCES ON
+  BATCH_SIZE 4
+
+  gemm_universal_f16n_f16t_f32t_tensor_op_f32_sm80.cu
+  gemm_f16n_f16n_f16t_tensor_op_f32_sm80.cu
+  gemm_f16n_f16n_f32n_tensor_op_f32_sm80.cu
+  gemm_f16n_f16n_f32t_tensor_op_f32_sm80.cu
+  gemm_f16n_f16t_f16t_tensor_op_f16_sm80.cu
+  gemm_f16n_f16t_f32t_tensor_op_f32_sm80.cu
+  gemm_f16t_f16n_f16t_tensor_op_f16_sm80.cu
+  gemm_f16t_f16n_f32t_tensor_op_f32_sm80.cu
+  gemm_f16t_f16t_f32n_tensor_op_f32_sm80.cu
+  gemm_f16t_f16t_f32t_tensor_op_f32_sm80.cu
+  gemm_bf16n_bf16n_f32t_tensor_op_f32_sm80.cu
+  gemm_bf16t_bf16t_bf16t_tensor_op_f32_sm80.cu
+)
+
+cutlass_test_unit_add_executable(
+  cutlass_test_unit_gemm_device_tensorop_f32_tf32_sm80
+
+  BATCH_SOURCES ON
+  BATCH_SIZE 4
+
+  gemm_tf32t_tf32n_f32t_tensor_op_f32_sm80.cu
+  gemm_tf32n_tf32t_f32t_tensor_op_f32_sm80.cu
+  gemm_tf32n_tf32n_f32t_tensor_op_f32_sm80.cu
+  gemm_tf32t_tf32t_f32t_tensor_op_f32_sm80.cu                 
+  gemm_universal_cf32n_cf32n_cf32n_tensor_op_f32_sm80.cu
+  gemm_cf32n_cf32t_cf32t_tensor_op_tf32_f32_sm80.cu
+  gemm_cf32t_cf32n_cf32t_tensor_op_tf32_f32_sm80.cu
+
+  gemm_f32n_f32n_f32t_tensor_op_f32_sm80.cu
+  gemm_f32n_f32n_f32t_tensor_op_bf16_f32_sm80.cu
+)
+
+cutlass_test_unit_add_executable(
+  cutlass_test_unit_gemm_device_tensorop_f64
+
+  BATCH_SOURCES ON
+  BATCH_SIZE 4
+
+  gemm_f64n_f64t_f64t_tensor_op_f64_sm80.cu
+  gemm_f64t_f64n_f64t_tensor_op_f64_sm80.cu
+
+  gemm_universal_cf64n_cf64t_cf64t_tensor_op_f64_sm80.cu                             
+  gemm_universal_cf64n_cf64t_cf64t_tensor_op_f64_gaussian_sm80.cu
+  gemm_cf64n_cf64t_cf64t_tensor_op_f64_sm80.cu
+  gemm_cf64t_cf64n_cf64t_tensor_op_f64_sm80.cu
+  gemm_cf64n_cf64t_cf64t_tensor_op_f64_gaussian_sm80.cu
+  gemm_cf64t_cf64n_cf64t_tensor_op_f64_gaussian_sm80.cu
+
+)
+
+cutlass_test_unit_add_executable(
+  cutlass_test_unit_gemm_device_tensorop_s32_sm80
+
+  BATCH_SOURCES ON
+  BATCH_SIZE 4
+
+  gemm_s8t_s8n_s32t_tensor_op_s32_sm80.cu
+  gemm_s8t_s8n_s32n_tensor_op_s32_sm80.cu
+  gemm_s8t_s8n_s8n_tensor_op_s32_sm80.cu
+  gemm_s8t_s8n_s8t_tensor_op_s32_sm80.cu
+  gemm_s4t_s4n_s32n_tensor_op_s32_sm80.cu
+  gemm_s4t_s4n_s32t_tensor_op_s32_sm80.cu
+  gemm_b1t_b1n_s32n_tensor_op_s32_sm80.cu
+  gemm_b1t_b1n_s32t_tensor_op_s32_sm80.cu
+
+  gemm_s8n_s8t_s8n_tensor_op_s32_sm80.cu
+  gemm_s4n_s4t_s4n_tensor_op_s32_sm80.cu
+)
+
+cutlass_test_unit_add_executable(
+  cutlass_test_unit_gemm_device_wmma
+
+  BATCH_SOURCES ON
+  BATCH_SIZE 4
 
   # wmma floating point tests
   gemm_f16t_f16n_f16t_wmma_tensor_op_f16_sm70.cu
@@ -222,5 +297,37 @@ cutlass_test_unit_add_executable(
   gemm_f16t_f16n_f16n_singlestage_wmma_tensor_op_f16_sm70.cu
 
   gemm_f16t_f16n_f32t_singlestage_wmma_tensor_op_f32_sm70.cu
-
 )
+
+cutlass_test_unit_add_executable(
+  cutlass_test_unit_gemm_device_tensorop_planar_complex
+
+  BATCH_SOURCES ON
+  BATCH_SIZE 4
+
+  gemm_planar_complex_f16_f16_f32_tensor_op_sm70.cu                              
+  gemm_planar_complex_f16_f16_f32_tensor_op_sm75.cu                                         
+  gemm_planar_complex_f16_f16_f32_tensor_op_sm80.cu
+)
+
+cutlass_test_unit_add_executable(
+  cutlass_test_unit_gemm_device_sparse_tensorop_sm80
+
+  BATCH_SOURCES ON
+  BATCH_SIZE 4
+
+  gemm_f16n_f16n_f16t_tensor_op_f32_sparse_sm80.cu
+  gemm_f16n_f16n_f32t_tensor_op_f32_sparse_sm80.cu
+  gemm_f16n_f16t_f16t_tensor_op_f16_sparse_sm80.cu
+  gemm_f16n_f16t_f32t_tensor_op_f32_sparse_sm80.cu
+  gemm_f16t_f16n_f16t_tensor_op_f16_sparse_sm80.cu
+  gemm_f16t_f16n_f32t_tensor_op_f32_sparse_sm80.cu
+  gemm_f16t_f16t_f32t_tensor_op_f32_sparse_sm80.cu
+  gemm_f32t_f32n_f32t_tensor_op_f32_sparse_sm80.cu
+  gemm_f32n_f32t_f32t_tensor_op_f32_sparse_sm80.cu
+  gemm_f32t_f32t_f32t_tensor_op_f32_sparse_sm80.cu
+  gemm_f32n_f32n_f32t_tensor_op_f32_sparse_sm80.cu
+  gemm_s8t_s8n_s32t_tensor_op_s32_sparse_sm80.cu
+  gemm_s4t_s4n_s32t_tensor_op_s32_sparse_sm80.cu
+)
+

test/unit/gemm/device/multistage_testbed.h

@@ -97,10 +97,45 @@ struct MultistageTestbed {
     return true;
   }
 
+  /// Waives test if CUDA device is insufficient
+  bool sufficient() const {
+    //
+    // Determine SMEM requirements and waive if not satisfied
+    //
+
+    int smem_size = int(sizeof(typename Gemm::GemmKernel::SharedStorage));
+
+    cudaDeviceProp properties;
+    int device_idx;
+    cudaError_t result = cudaGetDevice(&device_idx);
+
+    if (result != cudaSuccess) {
+      throw std::runtime_error("cudaGetDevice() API call failed.");
+    }
+
+    result = cudaGetDeviceProperties(&properties, device_idx);
+
+    if (result != cudaSuccess) {
+      throw std::runtime_error("cudaGetDeviceProperties() failed");
+    }
+
+    if (properties.sharedMemPerMultiprocessor < smem_size) {
+      return false;
+    }
+
+    return true;
+  }
+
   /// Executes one test
   bool run(cutlass::gemm::GemmCoord problem_size,
            ElementCompute alpha = ElementCompute(1),
            ElementCompute beta = ElementCompute(0)) {
+
+		// Waives test if CUDA device is insufficient
+		if (!sufficient()) {
+			return true;
+		}
+
     //
     // Allocate the GEMM workspace
     //
@@ -144,7 +179,11 @@ struct MultistageTestbed {
 
     cutlass::Status status = gemm_op.initialize(arguments);
 
-    EXPECT_TRUE(status == cutlass::Status::kSuccess);
+    if (status != cutlass::Status::kSuccess) {
+      cudaError_t error = cudaGetLastError();
+      std::cerr << "This test is not supported: " << cudaGetErrorString(error) << "\n";
+      return true;
+    }
 
     //
     // Run the GEMM

test/unit/gemm/device/simt_sgemm_nt_sm80.cu

@@ -39,7 +39,8 @@
 #include "cutlass/util/tensor_view_io.h"
 
 #include "testbed.h"
-
+    
+#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
 ////////////////////////////////////////////////////////////////////////////////
 
 TEST(SM80_Device_Gemm_f32n_f32t_f32t_simt_f32, 32x64x8_32x64x1) {
@@ -246,4 +247,8 @@ TEST(SM80_Device_Gemm_f32n_f32t_f32t_simt_f32, 128x256x8_64x64x1) {
   EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 }
 
-////////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
+
+/////////////////////////////////////////////////////////////////////////////////////////////////

test/unit/gemm/device/simt_sgemm_tn_sm80.cu

@@ -41,8 +41,10 @@
 
 #include "testbed.h"
 
+    
+#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
 ////////////////////////////////////////////////////////////////////////////////
-
+    
 TEST(SM80_Device_Gemm_f32t_f32n_f32t_simt_f32, 32x64x8_32x64x1) {
   
   using Element = float;
@@ -246,4 +248,8 @@ TEST(SM80_Device_Gemm_f32t_f32n_f32t_simt_f32, 128x256x8_64x64x1) {
   EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
 }
 
-////////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
+
+/////////////////////////////////////////////////////////////////////////////////////////////////

test/unit/gemm/device/testbed.h

@@ -247,6 +247,36 @@ struct Testbed {
     return compare_reference(problem_size, alpha, beta);
   }
 
+	/// Determine if the CUDA device is sufficient to run the kernel
+  bool sufficient() const {
+    //
+    // Determine SMEM requirements and waive if not satisfied
+    //
+
+    int smem_size = int(sizeof(typename Gemm::GemmKernel::SharedStorage));
+
+    cudaDeviceProp properties;
+    int device_idx;
+    cudaError_t result = cudaGetDevice(&device_idx);
+
+    if (result != cudaSuccess) {
+      throw std::runtime_error("cudaGetDevice() API call failed.");
+    }
+
+    result = cudaGetDeviceProperties(&properties, device_idx);
+
+    if (result != cudaSuccess) {
+      throw std::runtime_error("cudaGetDeviceProperties() failed");
+    }
+
+    if (properties.sharedMemPerMultiprocessor < smem_size) {
+      return false;
+    }
+
+    return true;
+  }
+
+
   /// Executes one test
   bool run(
     cutlass::gemm::GemmCoord problem_size, 
@@ -254,6 +284,10 @@ struct Testbed {
     ElementCompute alpha = ElementCompute(1), 
     ElementCompute beta = ElementCompute(0)) {
 
+		// Waive test if insufficient CUDA device
+		if (!sufficient()) {
+			return true;
+		}
 
     this->initialize(problem_size);
 
@@ -279,7 +313,11 @@ struct Testbed {
 
     cutlass::Status status = gemm_op.initialize(arguments, workspace.get());
 
-    EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
+    if (status != cutlass::Status::kSuccess) {
+      cudaError_t error = cudaGetLastError();
+      std::cerr << "This test is not supported: " << cudaGetErrorString(error) << "\n";
+      return true;
+    }
 
     //
     // Run the GEMM

test/unit/gemm/device/testbed_complex.h

@@ -31,6 +31,7 @@
 #include <iostream>
 #include <fstream>
 #include <sstream>
+#include <stdexcept>
 
 #include "../../common/cutlass_unit_test.h"
 
@@ -100,6 +101,34 @@ struct TestbedComplex : public Testbed<Gemm> {
     return this->compare_reference(problem_size, alpha, beta);
   }
 
+	bool sufficient() const {
+		//
+		// Determine SMEM requirements and waive if not satisfied
+		//
+
+		int smem_size = int(sizeof(typename Gemm::GemmKernel::SharedStorage));
+
+		cudaDeviceProp properties;
+		int device_idx;
+		cudaError_t result = cudaGetDevice(&device_idx);
+	
+		if (result != cudaSuccess) {
+			throw std::runtime_error("cudaGetDevice() API call failed.");
+		}
+		
+		result = cudaGetDeviceProperties(&properties, device_idx);
+
+		if (result != cudaSuccess) {
+			throw std::runtime_error("cudaGetDeviceProperties() failed");
+		}
+
+		if (properties.sharedMemPerMultiprocessor < smem_size) {
+			return false;
+		}
+
+		return true;
+	}
+
   /// Executes one test
   bool run(
     cutlass::gemm::GemmCoord problem_size, 
@@ -107,7 +136,17 @@ struct TestbedComplex : public Testbed<Gemm> {
     ElementCompute alpha = ElementCompute(1), 
     ElementCompute beta = ElementCompute(0)) {
 
+		// Waive the test if device not sufficient
+		if (!sufficient()) {
+			return true;
+		}
+
+		//
+		// Initialize workspace
+		//
+
     this->initialize(problem_size);
+		
 
     //
     // Initialize the GEMM operator

test/unit/gemm/device/testbed_interleaved.h

@@ -99,6 +99,35 @@ struct InterleavedTestbed {
       return false;
     }
 
+    return true;
+  }
+
+	/// Waives test if CUDA device is insufficient
+  bool sufficient() const {
+    //
+    // Determine SMEM requirements and waive if not satisfied
+    //
+
+    int smem_size = int(sizeof(typename Gemm::GemmKernel::SharedStorage));
+
+    cudaDeviceProp properties;
+    int device_idx;
+    cudaError_t result = cudaGetDevice(&device_idx);
+
+    if (result != cudaSuccess) {
+      throw std::runtime_error("cudaGetDevice() API call failed.");
+    }
+
+    result = cudaGetDeviceProperties(&properties, device_idx);
+
+    if (result != cudaSuccess) {
+      throw std::runtime_error("cudaGetDeviceProperties() failed");
+    }
+
+    if (properties.sharedMemPerMultiprocessor < smem_size) {
+      return false;
+    }
+
     return true;
   }
 
@@ -107,6 +136,10 @@ struct InterleavedTestbed {
     cutlass::gemm::GemmCoord problem_size, 
     ElementCompute alpha = ElementCompute(1), 
     ElementCompute beta = ElementCompute(0)) {
+
+		if (!sufficient()) {
+			return true;
+		}
     
     //
     // Allocate the GEMM workspace

test/unit/gemm/device/testbed_sparse.h

@@ -295,6 +295,34 @@ struct SparseTestbed {
     return compare_reference(problem_size, alpha, beta);
   }
 
+  bool sufficient() const {
+    //
+    // Determine SMEM requirements and waive if not satisfied
+    //
+
+    int smem_size = int(sizeof(typename Gemm::GemmKernel::SharedStorage));
+
+    cudaDeviceProp properties;
+    int device_idx;
+    cudaError_t result = cudaGetDevice(&device_idx);
+
+    if (result != cudaSuccess) {
+      throw std::runtime_error("cudaGetDevice() API call failed.");
+    }
+
+    result = cudaGetDeviceProperties(&properties, device_idx);
+
+    if (result != cudaSuccess) {
+      throw std::runtime_error("cudaGetDeviceProperties() failed");
+    }
+
+    if (properties.sharedMemPerMultiprocessor < smem_size) {
+      return false;
+    }
+
+    return true;
+  }
+
   /// Executes one test
   bool run(
     cutlass::gemm::GemmCoord problem_size, 
@@ -302,6 +330,11 @@ struct SparseTestbed {
     ElementCompute alpha = ElementCompute(1), 
     ElementCompute beta = ElementCompute(0)) {
 
+		// Waive test if insufficient CUDA device
+		if (!sufficient()) {
+			return true;
+		}
+
     this->initialize(problem_size);
 
     //
@@ -327,7 +360,10 @@ struct SparseTestbed {
 
     cutlass::Status status = gemm_op.initialize(arguments, workspace.get());
 
-    EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
+		// This failure is likely due to insufficient device capabilities. Waive the test.
+    if (status != cutlass::Status::kSuccess) {
+      return true;
+    }
 
     //
     // Run the GEMM

test/unit/gemm/device/testbed_universal.h

@@ -250,6 +250,34 @@ struct TestbedUniversal {
     return compare_reference(problem_size, alpha, beta);
   }
 
+  bool sufficient() const {
+    //
+    // Determine SMEM requirements and waive if not satisfied
+    //
+
+    int smem_size = int(sizeof(typename Gemm::GemmKernel::SharedStorage));
+
+    cudaDeviceProp properties;
+    int device_idx;
+    cudaError_t result = cudaGetDevice(&device_idx);
+
+    if (result != cudaSuccess) {
+      throw std::runtime_error("cudaGetDevice() API call failed.");
+    }
+
+    result = cudaGetDeviceProperties(&properties, device_idx);
+
+    if (result != cudaSuccess) {
+      throw std::runtime_error("cudaGetDeviceProperties() failed");
+    }
+
+    if (properties.sharedMemPerMultiprocessor < smem_size) {
+      return false;
+    }
+
+    return true;
+  }
+
   /// Executes one test
   bool run(
     cutlass::gemm::GemmUniversalMode mode,
@@ -258,6 +286,11 @@ struct TestbedUniversal {
     ElementCompute alpha = ElementCompute(1), 
     ElementCompute beta = ElementCompute(0)) {
 
+		// Waive test if insufficient CUDA device
+		if (!sufficient()) {
+			return true;
+		}
+
     this->initialize(problem_size);
 
     //

test/unit/gemm/threadblock/mma_multistage_sparse_testbed.h

@@ -328,19 +328,17 @@ struct SparseTestbed {
           test::gemm::threadblock::kernel_multistage_mma_sparse<Mma>,
           cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size);
 
-      EXPECT_EQ(result, cudaSuccess)
-          << " cudaFuncSetAttribute "
-             "cudaFuncAttributeMaxDynamicSharedMemorySize error: "
-          << cudaGetErrorString(result);
+      if (result != cudaSuccess) {
+          return true;
+      }
 
       result = cudaFuncSetAttribute(
           test::gemm::threadblock::kernel_multistage_mma_sparse<Mma>,
           cudaFuncAttributePreferredSharedMemoryCarveout, 100);
 
-      EXPECT_EQ(result, cudaSuccess)
-          << " cudaFuncSetAttribute "
-             "cudaFuncAttributePreferredSharedMemoryCarveout error: "
-          << cudaGetErrorString(result);
+      if (result != cudaSuccess) {
+          return true;
+      }
     }
 
     test::gemm::threadblock::kernel_multistage_mma_sparse<Mma>

test/unit/gemm/threadblock/mma_multistage_testbed.h

@@ -266,19 +266,17 @@ struct Testbed {
           test::gemm::threadblock::kernel_multistage_mma<Mma>,
           cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size);
 
-      EXPECT_EQ(result, cudaSuccess)
-          << " cudaFuncSetAttribute "
-             "cudaFuncAttributeMaxDynamicSharedMemorySize error: "
-          << cudaGetErrorString(result);
+      if (result != cudaSuccess) {
+        return true;
+      }
 
       result = cudaFuncSetAttribute(
           test::gemm::threadblock::kernel_multistage_mma<Mma>,
           cudaFuncAttributePreferredSharedMemoryCarveout, 100);
 
-      EXPECT_EQ(result, cudaSuccess)
-          << " cudaFuncSetAttribute "
-             "cudaFuncAttributePreferredSharedMemoryCarveout error: "
-          << cudaGetErrorString(result);
+      if (result != cudaSuccess) {
+          return true;
+      }
     }
 
     test::gemm::threadblock::kernel_multistage_mma<Mma>

test/unit/gemm/warp/gemm_sm70.cu

@@ -199,6 +199,91 @@ TEST(SM70_warp_gemm_tensor_op_crosswise, 64x64x32_64x64x32_16x16x4) {
 
   test::gemm::warp::Testbed<MmaTensorOp, cutlass::gemm::GemmShape<64, 64, 32> >().run();
 }
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(SM70_warp_gemm_volta_tensor_op_canonical_f32_row_col, 64x64x16_64x64x4_8x8x4) {
+  
+  using Shape = cutlass::gemm::GemmShape<64, 64, 4>;
+  using InstructionShape = cutlass::gemm::GemmShape<8, 8, 4>;
+  using ElementA = cutlass::half_t;
+  using ElementB = cutlass::half_t;
+  using ElementC = float;
+  using LayoutA = cutlass::layout::RowMajor;
+  using LayoutB = cutlass::layout::ColumnMajor;
+
+  using Policy = cutlass::gemm::warp::MmaTensorOpPolicy<
+    cutlass::arch::Mma<
+      cutlass::gemm::GemmShape<16, 16, 4>,
+      32,
+      ElementA,
+      cutlass::layout::RowMajor,
+      ElementB,
+      cutlass::layout::ColumnMajor,
+      ElementC,
+      cutlass::layout::RowMajor,
+      cutlass::arch::OpMultiplyAdd
+    >,
+    cutlass::MatrixShape<1, 1>
+  >;
+
+  using MmaTensorOp = cutlass::gemm::warp::MmaVoltaTensorOp<
+    Shape,
+    ElementA,
+    LayoutA,
+    ElementB,
+    LayoutB,
+    ElementC,
+    cutlass::layout::RowMajor,
+    Policy
+  >;
+
+  test::gemm::warp::Testbed<MmaTensorOp,
+                            cutlass::gemm::GemmShape<64, 64, 16> >()
+      .run();
+}
+
+TEST(SM70_warp_gemm_volta_tensor_op_canonical_f32_col_row, 64x64x16_64x64x4_8x8x4) {
+  
+  using Shape = cutlass::gemm::GemmShape<64, 64, 4>;
+  using InstructionShape = cutlass::gemm::GemmShape<8, 8, 4>;
+  using ElementA = cutlass::half_t;
+  using ElementB = cutlass::half_t;
+  using ElementC = float;
+  using LayoutA = cutlass::layout::ColumnMajor;
+  using LayoutB = cutlass::layout::RowMajor;
+
+  using Policy = cutlass::gemm::warp::MmaTensorOpPolicy<
+    cutlass::arch::Mma<
+      cutlass::gemm::GemmShape<16, 16, 4>,
+      32,
+      ElementA,
+      LayoutA,
+      ElementB,
+      LayoutB,
+      ElementC,
+      cutlass::layout::RowMajor,
+      cutlass::arch::OpMultiplyAdd
+    >,
+    cutlass::MatrixShape<1, 1>
+  >;
+
+  using MmaTensorOp = cutlass::gemm::warp::MmaVoltaTensorOp<
+    Shape,
+    ElementA,
+    LayoutA,
+    ElementB,
+    LayoutB,
+    ElementC,
+    cutlass::layout::RowMajor,
+    Policy
+  >;
+
+  test::gemm::warp::Testbed<MmaTensorOp,
+                            cutlass::gemm::GemmShape<64, 64, 16> >()
+      .run();
+}
+
 /////////////////////////////////////////////////////////////////////////////////////////////////
 
 #endif // CUTLASS_ARCH_MMA_SM70_SUPPORTED

test/unit/gemm/warp/testbed.h

@@ -30,6 +30,7 @@
 
 #include "cutlass/cutlass.h"
 #include "cutlass/aligned_buffer.h"
+#include "cutlass/numeric_types.h"
 #include "cutlass/subbyte_reference.h"
 #include "cutlass/platform/platform.h"
 
@@ -1019,9 +1020,11 @@ __global__ void sparse_kernel(
     typename Mma::ElementB, ThreadblockShape::kN * ThreadblockShape::kK> smem_buffer_B;
 
   __shared__ cutlass::AlignedBuffer<
-      typename Mma::ElementE, ThreadblockShape::kM * ThreadblockShape::kK /
+      typename Mma::ElementE, Mma::Shape::kM * Mma::Shape::kK /
                                   Mma::kSparse / Mma::kElementsPerElementE>
       smem_buffer_E;
+  
+  __syncthreads();
 
   if (threadIdx.x == 0) {
     typename Mma::ElementA *smem_ptr_A = smem_buffer_A.data();
@@ -1168,6 +1171,7 @@ struct SparseTestbed {
 
   /// Allocates workspace in device memory
   SparseTestbed() {
+
     tensor_A.reset(cutlass::make_Coord(ThreadblockShape::kM,
                                        ThreadblockShape::kK / Sparse));
     tensor_A_uncompressed.reset(