Cutlass 1.3 Release (#42)

CUTLASS 1.3 Release
- Efficient GEMM kernel targeting Volta Tensor Cores via mma.sync instruction added in CUDA 10.1.

tools/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification, are permitted
 # provided that the following conditions are met:

tools/nvrtc/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification, are permitted
 # provided that the following conditions are met:

tools/nvrtc/cutlass/nvrtc/environment.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/nvrtc/stdlib/stdint.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification, are permitted
 # provided that the following conditions are met:

tools/test/perf/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification, are permitted
 # provided that the following conditions are met:
@@ -32,6 +32,8 @@ set(CUTLASS_PERF_TEST_HEADERS
   gemm/cutlass_dispatch_splitK_PI.h
   gemm/gemm_perf_testbed.h
   gemm/gemm_profiler.h
+  gemm/cutlass_volta884_dispatch.h
+  gemm/cutlass_volta884_dispatch_splitK_PI.h
 )
 
 set(CUTLASS_PERF_TEST_SOURCES
@@ -43,8 +45,14 @@ set(CUTLASS_PERF_TEST_SOURCES
   gemm/igemm.cu
   gemm/igemm_splitK.cu
   gemm/wmma_gemm.cu
-  gemm/wmma_binary_gemm.cu
-  gemm/wmma_integer_gemm.cu
+  gemm/volta884_gemm.cu                       
+  gemm/volta884_gemm_splitK.cu
+  gemm/volta884_gemm_cta_rasterization_tn.cu
+  gemm/volta884_gemm_cta_rasterization_tt.cu
+  gemm/volta884_gemm_cta_rasterization_nn.cu
+  gemm/volta884_gemm_cta_rasterization_nt.cu
+  gemm/wmma_binary_gemm.cu                         
+  gemm/wmma_integer_gemm.cu                        
 )
 
 source_group("Source\ Files" FILES ${CUTLASS_PERF_TEST_SOURCES})
@@ -62,5 +70,7 @@ cutlass_add_executable(
     ${CUTLASS_PERF_TEST_HEADERS}
 )
 
-target_link_libraries(cutlass_perf_test ${CUBLAS_LIBRARY})
+if(CUTLASS_ENABLE_CUBLAS)
+  target_link_libraries(cutlass_perf_test ${CUBLAS_LIBRARY})
+endif()
 

tools/test/perf/cutlass_perf_test.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/perf/cutlass_perf_test.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -32,6 +32,14 @@
 #include "tools/test/perf/testbench_output.h"
 #include "tools/test/perf/gemm/gemm_profiler.h"
 
+#if !defined(CUTLASS_ENABLE_CUBLAS)
+#define CUTLASS_ENABLE_CUBLAS 0
+#endif
+
+#if !defined(CUTLASS_ENABLE_CUDNN)
+#define CUTLASS_ENABLE_CUDNN 0
+#endif
+
 namespace perf {
 
 typedef int (GemmProfileFunc)(

tools/test/perf/gemm/cublas_dispatch.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -67,6 +67,7 @@ struct CublasGemmDispatch {
                             CDeviceType *C,
                             int ldc,
                             cublasGemmAlgo_t algorithm) {
+    #if CUTLASS_ENABLE_CUBLAS
     return cublasGemmEx(handle,
                         convert(layout_a),
                         convert(layout_b),
@@ -86,6 +87,9 @@ struct CublasGemmDispatch {
                         ldc,
                         cutlass::TypeTraits<AccumulatorDeviceType>::cublas_type,
                         algorithm);
+    #else
+    return CUBLAS_STATUS_NOT_SUPPORTED;
+    #endif
   }
 };
 
@@ -131,7 +135,7 @@ struct CublasBatchedStridedGemmDispatch {
     long long int batch_stride_C,
     int batch_count,
     cublasGemmAlgo_t algorithm) {
-#if defined(CUDA_VERSION) && CUDA_VERSION >= 9010
+    #if CUTLASS_ENABLE_CUBLAS && defined(CUDA_VERSION) && CUDA_VERSION >= 9010
     return cublasGemmStridedBatchedEx(handle,
       convert(layout_a),
       convert(layout_b),
@@ -155,9 +159,9 @@ struct CublasBatchedStridedGemmDispatch {
       batch_count,
       cutlass::TypeTraits<AccumulatorDeviceType>::cublas_type,
       algorithm);
-#else
+    #else
     return CUBLAS_STATUS_NOT_SUPPORTED;
-#endif
+    #endif
   }
 };
 

tools/test/perf/gemm/cutlass_dispatch.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -33,7 +33,12 @@ template <typename Gemm_,
           typename Compute_,
           typename ScalarEpilogue_,
           bool ThreadMultiplyAdd_,
-          bool RunCuBLAS_ = true>
+          #if CUTLASS_ENABLE_CUBLAS
+          bool RunCuBLAS_ = true
+          #else
+          bool RunCuBLAS_ = false
+          #endif
+>
 struct CutlassDispatch {
   typedef typename Gemm_::Params Params;
   typedef Gemm_ Gemm;
@@ -131,8 +136,6 @@ struct CutlassDispatchBasic {
   typedef typename Traits::ScalarC ScalarC;
   /// The scalar for D.
   typedef typename Traits::ScalarD ScalarD;
-
-  // TODO - support alternative accumulator and scalar types
   typedef ScalarD Compute;
   typedef Compute ScalarEpilogue;
 

tools/test/perf/gemm/cutlass_dispatch_splitK_PI.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
-* Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+* Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification, are permitted
 * provided that the following conditions are met:
@@ -40,8 +40,13 @@ template <typename KernelClass_,
     typename Compute_,
     typename ScalarEpilogue_,
     bool ThreadMultiplyAdd_,
-    bool RunCuBLAS_ = true>
-  struct CutlassDispatchSplitKPIGemm {
+    #if CUTLASS_ENABLE_CUBLAS
+    bool RunCuBLAS_ = true
+    #else
+    bool RunCuBLAS_ = false
+    #endif
+>
+struct CutlassDispatchSplitKPIGemm {
   typedef typename KernelClass_::Params Params;
   typedef KernelClass_ KernelClass;
   typedef Index_ Index;
@@ -87,8 +92,21 @@ template <typename KernelClass_,
     ScalarD* d_d,
     Index ldd) {
     params.init_problem(m, n, k);
-    int workspace_size_in_byte = params.required_workspace_memory_in_byte();
-    
+    size_t workspace_size_in_byte = params.required_workspace_memory_in_byte();
+    size_t available_device_memory_in_byte = 0;
+    size_t device_memory_in_byte = 0;
+    cudaError_t cudaMemGetInfo_err = cudaMemGetInfo(&available_device_memory_in_byte, &device_memory_in_byte);
+    if (cudaMemGetInfo_err != cudaSuccess) {
+      std::cout << "\ncudaMemGetInfo error: " << cudaGetErrorString(cudaMemGetInfo_err)
+        << "\n";
+    }
+
+    if (workspace_size_in_byte > available_device_memory_in_byte) {
+      std::cout << "reqested workspace memory size("<< workspace_size_in_byte << 
+                   ") is larger than available memory size("<< available_device_memory_in_byte << "). Abort." << std::endl;
+      throw std::runtime_error("reqested workspace memory size is larger than available memory size. Abort.");
+    }
+
     cudaError_t workspace_err = cudaMalloc(&workspace_ptr, workspace_size_in_byte);
     if (workspace_err != cudaSuccess) {
       std::cout << "\nCUDA workspace malloc error: " << cudaGetErrorString(workspace_err)
@@ -153,8 +171,6 @@ struct CutlassDispatchSplitKPIGemmBasic {
   typedef typename Traits::ScalarC ScalarC;
   /// The scalar for D.
   typedef typename Traits::ScalarD ScalarD;
-
-  // TODO - support alternative accumulator and scalar types
   typedef ScalarD Compute;
   typedef Compute ScalarEpilogue;
 

tools/test/perf/gemm/cutlass_volta884_dispatch.h

@@ -0,0 +1,113 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma once
+
+
+#include "cutlass/gemm/gemm.h"
+#include "cutlass/gemm/volta884_gemm_traits.h"
+
+#include "tools/test/perf/cutlass_perf_test.h"
+#include "tools/test/perf/gemm/gemm_profiler.h"
+#include "tools/test/perf/gemm/cutlass_dispatch.h"
+#include "tools/test/perf/gemm/gemm_perf_testbed.h"
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename Traits>
+struct Volta884GemmDispatch {
+
+  typedef cutlass::gemm::Gemm<Traits> Gemm;
+
+  typedef typename Gemm::Params Params;
+
+  typedef typename Traits::ScalarC ScalarC;
+  typedef typename Traits::ScalarD ScalarD;
+  typedef typename Traits::ScalarD ScalarEpilogue;
+
+  /// Indicate warp-level GEMM
+  static bool const kThreadMultiplyAdd = false;
+
+  #if CUTLASS_ENABLE_CUBLAS
+  static bool const kRunCuBLAS = true;
+  #else
+  static bool const kRunCuBLAS = false;
+  #endif
+
+  static cutlass::MatrixLayout::Kind const kLayoutA = Traits::kLayoutA;
+  static cutlass::MatrixLayout::Kind const kLayoutB = Traits::kLayoutB;
+
+  //
+  // Data members
+  //
+
+  /// Params argument
+  Params params;
+
+  //
+  // Methods
+  //
+
+  Volta884GemmDispatch() {}
+
+  /// Initializes params object
+  Volta884GemmDispatch(int m, int n, int k, ScalarEpilogue alpha, half const* d_a, int lda,
+                  half const* d_b, int ldb, ScalarEpilogue beta, ScalarC const* d_c, int ldc,
+                  ScalarD* d_d, int ldd) {
+
+    params.initialize(m, n, k, alpha, d_a, lda, d_b, ldb, beta, d_c, ldc, d_d, ldd);
+  }
+  
+  Volta884GemmDispatch(int m,
+    int n,
+    int k,
+    ScalarEpilogue alpha,
+    half const* d_a,
+    int lda,
+    long long int batch_stride_A,
+    half const* d_b,
+    int ldb,
+    long long int batch_stride_B,
+    ScalarEpilogue beta,
+    ScalarC const* d_c,
+    int ldc,
+    long long int batch_stride_C,
+    ScalarD* d_d,
+    int ldd,
+    long long int batch_stride_D,
+    int batch_count) {
+    assert(0);//not yet supported
+  }
+
+  /// Initializes params object
+  Volta884GemmDispatch(Params const& _params) : params(_params) {}
+
+  /// Launches kernel
+  cudaError_t operator()() { return Gemm::launch(params); }
+};

tools/test/perf/gemm/cutlass_volta884_dispatch_splitK_PI.h

@@ -0,0 +1,139 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma once
+
+
+#include "cutlass/gemm/device_gemm.h"
+#include "cutlass/gemm/volta884_gemm_traits.h"
+
+#include "tools/test/perf/cutlass_perf_test.h"
+#include "tools/test/perf/gemm/gemm_profiler.h"
+#include "tools/test/perf/gemm/cutlass_dispatch.h"
+#include "tools/test/perf/gemm/gemm_perf_testbed.h"
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename Traits>
+struct Volta884GemmDispatchSplitKPI {
+
+  typedef cutlass::gemm::DeviceGemm<Traits> Gemm;
+
+  typedef typename Gemm::Params Params;
+
+  typedef typename Traits::ScalarC ScalarC;
+  typedef typename Traits::ScalarD ScalarD;
+  typedef typename Traits::Scalar ScalarEpilogue;
+
+  /// Indicate warp-level GEMM
+  static bool const kThreadMultiplyAdd = false;
+
+  #if CUTLASS_ENABLE_CUBLAS
+  static bool const kRunCuBLAS = true;
+  #else
+  static bool const kRunCuBLAS = false;
+  #endif
+
+  static cutlass::MatrixLayout::Kind const kLayoutA = Traits::kLayoutA;
+  static cutlass::MatrixLayout::Kind const kLayoutB = Traits::kLayoutB;
+
+  //
+  // Data members
+  //
+
+  /// Params argument
+  Params params;
+
+  /// splitK PI require workspace
+  typename cutlass::TypeTraits<typename Traits::ScalarAccum>::device_type *workspace_ptr;
+
+  //
+  // Methods
+  //
+
+  Volta884GemmDispatchSplitKPI() {}
+
+  /// Initializes params object
+  Volta884GemmDispatchSplitKPI(int m, int n, int k, ScalarEpilogue alpha, half const* d_a, int lda,
+                  half const* d_b, int ldb, ScalarEpilogue beta, ScalarC const* d_c, int ldc,
+                  ScalarD* d_d, int ldd) {
+    params.init_problem(m, n, k);
+    size_t workspace_size_in_byte = params.required_workspace_memory_in_byte();
+    size_t available_device_memory_in_byte = 0;
+    size_t device_memory_in_byte = 0;
+    cudaError_t cudaMemGetInfo_err = cudaMemGetInfo(&available_device_memory_in_byte, &device_memory_in_byte);
+    if (cudaMemGetInfo_err != cudaSuccess) {
+      std::cout << "\ncudaMemGetInfo error: " << cudaGetErrorString(cudaMemGetInfo_err)
+        << "\n";
+    }
+
+    if (workspace_size_in_byte > available_device_memory_in_byte) {
+      std::cout << "reqested workspace memory size(" << workspace_size_in_byte <<
+        ") is larger than available memory size(" << available_device_memory_in_byte << "). Abort." << std::endl;
+      throw std::runtime_error("reqested workspace memory size is larger than available memory size. Abort.");
+    }
+
+    cudaError_t workspace_err = cudaMalloc(&workspace_ptr, workspace_size_in_byte);
+    if (workspace_err != cudaSuccess) {
+      std::cout << "\nCUDA workspace malloc error: " << cudaGetErrorString(workspace_err)
+        << "\n";
+    }
+
+    params.initialize(alpha, d_a, lda, d_b, ldb, beta, d_c, ldc, d_d, ldd, workspace_ptr, 8 /*volta884 requires leading dim to be mulitiple of 8*/);
+  }
+
+  Volta884GemmDispatchSplitKPI(int m,
+    int n,
+    int k,
+    ScalarEpilogue alpha,
+    half const* d_a,
+    int lda,
+    long long int batch_stride_A,
+    half const* d_b,
+    int ldb,
+    long long int batch_stride_B,
+    ScalarEpilogue beta,
+    ScalarC const* d_c,
+    int ldc,
+    long long int batch_stride_C,
+    ScalarD* d_d,
+    int ldd,
+    long long int batch_stride_D,
+    int batch_count) {
+    assert(0);//not yet supported
+  }
+
+  /// Initializes params object
+  Volta884GemmDispatchSplitKPI(Params const& _params) : params(_params) {}
+
+  /// Launches kernel
+  cudaError_t operator()() { 
+    return Gemm::launch(params); 
+  }
+};

tools/test/perf/gemm/dgemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/perf/gemm/gemm_perf_testbed.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -284,10 +284,14 @@ class GemmTestbed {
   /// Constructs a basic workspace
   GemmTestbed(InitialDistribution const &_dist = InitialDistribution())
       : initial_distribution(_dist) {
+    #if CUTLASS_ENABLE_CUBLAS
     status = cublasCreate(&handle);
     if (status != CUBLAS_STATUS_SUCCESS) {
       throw cutlass::cuda_exception("Failed to create CUBLAS handle");
     }
+    #else
+    status = CUBLAS_STATUS_NOT_INITIALIZED;
+    #endif
   }
 
   /// Constructs a workspace for verifying GEMM, assumes
@@ -296,15 +300,26 @@ class GemmTestbed {
               cublasGemmAlgo_t algorithm_ = CUBLAS_GEMM_DEFAULT,
               InitialDistribution const &_dist = InitialDistribution())
       : problem(_problem), initial_distribution(_dist) {
+    #if CUTLASS_ENABLE_CUBLAS
     status = cublasCreate(&handle);
     if (status != CUBLAS_STATUS_SUCCESS) {
       throw cutlass::cuda_exception("Failed to create CUBLAS handle");
     }
+    #else
+    status = CUBLAS_STATUS_NOT_INITIALIZED;
+    #endif
 
     resize(problem);
   }
 
-  ~GemmTestbed() { status = cublasDestroy(handle); }
+  /// Destructs the GEMM testbed
+  ~GemmTestbed() {
+    #if CUTLASS_ENABLE_CUBLAS
+    if (status != CUBLAS_STATUS_NOT_INITIALIZED) {
+      status = cublasDestroy(handle);
+    }
+    #endif
+  }
 
   /// Returns true if the last CUBLAS call returned successfully
   bool good() const { return status == CUBLAS_STATUS_SUCCESS; }
@@ -388,6 +403,7 @@ class GemmTestbed {
 
   /// Launches the cuBLAS GEMM - does not initialize output matrix
   cublasStatus_t launch_cublas(cublasGemmAlgo_t algo) {
+    #if CUTLASS_ENABLE_CUBLAS
     if (problem.batch_count == 1) {
       CublasDispatch dispatch;
 
@@ -441,6 +457,9 @@ class GemmTestbed {
 
       return status;
     }
+    #else
+    return CUBLAS_STATUS_NOT_SUPPORTED;
+    #endif
   }
 
   /// Verifies the 'test' tensor with 'ref'

tools/test/perf/gemm/gemm_profiler.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -146,22 +146,23 @@ class GemmProfiler {
       , kernel_name
       , problem
     );
-
+    
+    result.disposition = Disposition::NotVerified;
+    
     if (options.dry_run) {
       result.disposition = Disposition::NotRun;
       return result;
     }
 
     if (CutlassDispatch::kRunCuBLAS) {
+#if CUTLASS_ENABLE_CUBLAS
       testbed.compute_reference(algorithm);
 
       if (cudaDeviceSynchronize() != cudaSuccess) {
         result.disposition = Disposition::NotVerified;
         return result;
       }
-    }
-    else {
-      result.disposition = Disposition::Passed;
+#endif
     }
 
     CutlassDispatch *dispatch_ptr;
@@ -214,11 +215,13 @@ class GemmProfiler {
     }
 
     if (CutlassDispatch::kRunCuBLAS) {
+#if CUTLASS_ENABLE_CUBLAS
       if (testbed.verify_with_reference()) {
         result.disposition = Disposition::Passed;
       } else {
         result.disposition = Disposition::Incorrect;
       }
+#endif
     }
 
     if (options.save_workspace(result.disposition == Disposition::Passed)) {
@@ -270,11 +273,34 @@ class GemmProfiler {
     result.runtime = double(average_ms) / double(options.iterations);
     result.gflops = testbed.GFLOPs_per_sec(result.runtime);
 
-    if (result.disposition != Disposition::Passed) {
-      std::cout << "[\033[1;31mFAILED\033[0m]: " << kernel_name
-                << " failed with disposition: " << result.disposition << "\n";
+    if (result.disposition == Disposition::Unknown) {
+      std::cout << "[\033[1;30mUnknown\033[0m]: " << kernel_name
+                << " with disposition: " << result.disposition << "\n";
+    }
+    if (result.disposition == Disposition::NotRun) {
+      std::cout << "[\033[1;33mNotRun\033[0m]: " << kernel_name
+                << " with disposition: " << result.disposition << "\n";
+    }
+    if (result.disposition == Disposition::Passed) {
+      std::cout << "[\033[1;32mPassed\033[0m]: " << kernel_name
+                << " with disposition: " << result.disposition << "\n";
+    }
+    if (result.disposition == Disposition::Incorrect) {
+      std::cout << "[\033[1;31mIncorrect\033[0m]: " << kernel_name
+                << " with disposition: " << result.disposition << "\n";
+    }
+    if (result.disposition == Disposition::Failed) {
+      std::cout << "[\033[1;31mFailed\033[0m]: " << kernel_name
+                << " with disposition: " << result.disposition << "\n";
+    }
+    if (result.disposition == Disposition::NotVerified) {
+      std::cout << "[\033[1;34mNotVerified\033[0m]: " << kernel_name
+                << " with disposition: " << result.disposition << "\n";
+    }
+    if (result.disposition == Disposition::Invalid) {
+      std::cout << "[\033[1;36mInvalid\033[0m]: " << kernel_name
+                << " with disposition: " << result.disposition << "\n";
     }
-
     delete dispatch_ptr;
     return result;
   }
@@ -299,7 +325,7 @@ class GemmProfiler {
 
     std::vector<PerformanceResult<GemmProblem> > results;
 
-    results.push_back(execute_cutlass<CutlassDispatch>(problem, algorithm));
+      results.push_back(execute_cutlass<CutlassDispatch>(problem, algorithm));
     // cool-down period
     if (!options.dry_run) {
       pause(options.sleep_time);
@@ -402,10 +428,10 @@ int profile_gemm(TestbenchOutput<GemmProblem> &output,
     GemmProfiler perf(output, kernel, cutlass_algo, options, config);
     if (options.peak_performance) {
       perf.template peak<Dispatch>(
-          config.problem_range.M, config.problem_range.N, config.problem_range.K);
+          config.gemm_problem_range.M, config.gemm_problem_range.N, config.gemm_problem_range.K);
     } else {
       perf.template schmoo<Dispatch>(
-          config.problem_range.M, config.problem_range.N, config.problem_range.K, config.problem_range.batch_count);
+          config.gemm_problem_range.M, config.gemm_problem_range.N, config.gemm_problem_range.K, config.gemm_problem_range.batch_count);
     }
   }
 

tools/test/perf/gemm/hgemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/perf/gemm/igemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -23,6 +23,8 @@
  *
  **************************************************************************************************/
 
+#if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))
+
 #include "cutlass/gemm/gemm.h"
 #include "cutlass/gemm/igemm_traits.h"
 #include "tools/test/perf/cutlass_perf_test.h"
@@ -36,6 +38,7 @@ namespace perf {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
+template <typename DummyT>
 int profile_igemm(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
 
   typedef perf::GemmProfiler<int8_t, int8_t, int, int, int> GemmProfiler;
@@ -91,6 +94,21 @@ int profile_igemm(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &
     results |= profile_gemm<Dispatch, GemmProfiler>(output, "igemm_tt", options, config);
   }
 
+  return results;
+}
+
+template <typename DummyT>
+int profile_igemm_32x32x128(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
+
+  typedef perf::GemmProfiler<int8_t, int8_t, int, int, int> GemmProfiler;
+
+  // compute capability check
+  if (!options.compute_capability(6, 1)) {
+    return 0;
+  }
+
+  int results = 0;
+
   {
     typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
             cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<128, 32, 32>, int,
@@ -138,8 +156,18 @@ int profile_igemm(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &
   return results;
 }
 
+
+
 struct IgemmRegistrar {
-  IgemmRegistrar() { RegisterGemmProfileFunc(profile_igemm); }
+  IgemmRegistrar()
+  {
+    RegisterGemmProfileFunc(profile_igemm<void>);
+
+#ifdef EXHAUSTIVE_PROF
+    RegisterGemmProfileFunc(profile_igemm_32x32x128<void>);
+#endif // defined EXHAUSTIVE_PROF
+
+  }
 };
 
 volatile IgemmRegistrar _IgemmRegistrar;
@@ -147,3 +175,5 @@ volatile IgemmRegistrar _IgemmRegistrar;
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 } // namespace perf
+
+#endif // if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))

tools/test/perf/gemm/igemm_splitK.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -23,6 +23,8 @@
  *
  **************************************************************************************************/
 
+#if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))
+
 #include "cutlass/gemm/gemm.h"
 #include "cutlass/gemm/igemm_traits.h"
 #include "cutlass/reduction/batched_reduction_traits.h"
@@ -154,7 +156,6 @@ int profile_igemm_splitkpi_kernel(
     results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_tt", options, config, algo + "_splitk_pi");
   }
 
-
   return results;
 }
 
@@ -200,3 +201,5 @@ volatile IgemmSplitKPIRegistrar _IgemmSplitKPIRegistrar;
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 } // namespace perf
+
+#endif // if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))

tools/test/perf/gemm/sgemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -38,8 +38,10 @@ namespace perf {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
+
+/// Profile simple gemm kernels
 template <typename OutputTile>
-int profile_sgemm_kernel(
+int profile_simple_sgemm_kernel(
   TestbenchOutput<GemmProblem> &output,
   TestbenchOptions const &options,
   Config const &config,
@@ -98,6 +100,24 @@ int profile_sgemm_kernel(
     results |= profile_gemm<Dispatch, SGemmProfiler>(output, name + "_tt", options, config, algo);
   }
 
+  return results;
+}
+
+
+
+/// Profile swizzle-raster gemm kernels
+template <typename OutputTile>
+int profile_swizzle_sgemm_kernel(
+  TestbenchOutput<GemmProblem> &output,
+  TestbenchOptions const &options,
+  Config const &config,
+  std::string const &name,
+  std::string const &algo) {
+
+  typedef perf::GemmProfiler<float, float, float, float, float> SGemmProfiler;
+
+  int results = 0;
+
   {
     typedef int index;
     typedef cutlass::gemm::SgemmConfig<OutputTile,
@@ -1321,7 +1341,11 @@ int profile_sgemm_kernel(
 int profile_sgemm(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
   int results = 0;
 
-  results |= profile_sgemm_kernel<cutlass::Shape<8, 128, 128> >(output, options, config, "sgemm", "128x128");
+  results |= profile_simple_sgemm_kernel<cutlass::Shape<8, 128, 128> >(output, options, config, "sgemm", "128x128");
+
+#ifdef EXHAUSTIVE_PROF
+  results |= profile_swizzle_sgemm_kernel<cutlass::Shape<8, 128, 128> >(output, options, config, "sgemm", "128x128");
+#endif // defined EXHAUSTIVE_PROF
 
   return results;
 }

tools/test/perf/gemm/sgemm_splitK.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -38,7 +38,7 @@ namespace perf {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template <typename OutputTile, int splits_count>
+template <typename OutputTile, typename threadGemmShape, typename threadReductionShape, int splits_count>
 int profile_sgemm_splitkpi_kernel(
   TestbenchOutput<GemmProblem> &output,
   TestbenchOptions const &options,
@@ -53,7 +53,8 @@ int profile_sgemm_splitkpi_kernel(
   {
     /*batched sgemm traits*/
     typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
-      cutlass::MatrixLayout::kColumnMajor, OutputTile>
+      cutlass::MatrixLayout::kColumnMajor, OutputTile,
+      cutlass::gemm::LinearScaling<float>, threadGemmShape>
       SgemmTraits;
     /*batched reduction traits*/
     typedef cutlass::reduction::BatchedReductionTraits<float,
@@ -64,7 +65,7 @@ int profile_sgemm_splitkpi_kernel(
       splits_count,
       cutlass::Shape<1, 1, 128>,
       cutlass::Shape<1, 1, 64>,
-      cutlass::Shape<1, 1, 2> >
+      threadReductionShape >
       BatchedReductionTraits;
 
     // create a device gemm 
@@ -77,7 +78,8 @@ int profile_sgemm_splitkpi_kernel(
   {
     /*batched sgemm traits*/
     typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
-      cutlass::MatrixLayout::kRowMajor, OutputTile>
+      cutlass::MatrixLayout::kRowMajor, OutputTile,
+      cutlass::gemm::LinearScaling<float>, threadGemmShape>
       SgemmTraits;
     /*batched reduction traits*/
     typedef cutlass::reduction::BatchedReductionTraits<float,
@@ -88,7 +90,7 @@ int profile_sgemm_splitkpi_kernel(
       splits_count,
       cutlass::Shape<1, 1, 128>,
       cutlass::Shape<1, 1, 64>,
-      cutlass::Shape<1, 1, 2> >
+      threadReductionShape >
       BatchedReductionTraits;
 
     // create a device gemm 
@@ -101,7 +103,8 @@ int profile_sgemm_splitkpi_kernel(
   {
     /*batched sgemm traits*/
     typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
-      cutlass::MatrixLayout::kColumnMajor, OutputTile>
+      cutlass::MatrixLayout::kColumnMajor, OutputTile,
+      cutlass::gemm::LinearScaling<float>, threadGemmShape>
       SgemmTraits;
     /*batched reduction traits*/
     typedef cutlass::reduction::BatchedReductionTraits<float,
@@ -112,7 +115,7 @@ int profile_sgemm_splitkpi_kernel(
       splits_count,
       cutlass::Shape<1, 1, 128>,
       cutlass::Shape<1, 1, 64>,
-      cutlass::Shape<1, 1, 2> >
+      threadReductionShape >
       BatchedReductionTraits;
 
     // create a device gemm 
@@ -125,7 +128,8 @@ int profile_sgemm_splitkpi_kernel(
   {
     /*batched sgemm traits*/
     typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
-      cutlass::MatrixLayout::kRowMajor, OutputTile>
+      cutlass::MatrixLayout::kRowMajor, OutputTile,
+      cutlass::gemm::LinearScaling<float>, threadGemmShape>
       SgemmTraits;
     /*batched reduction traits*/
     typedef cutlass::reduction::BatchedReductionTraits<float,
@@ -136,7 +140,7 @@ int profile_sgemm_splitkpi_kernel(
       splits_count,
       cutlass::Shape<1, 1, 128>,
       cutlass::Shape<1, 1, 64>,
-      cutlass::Shape<1, 1, 2> >
+      threadReductionShape >
       BatchedReductionTraits;
 
     // create a device gemm 
@@ -153,25 +157,143 @@ int profile_sgemm_splitkpi_kernel(
 /// Profiles all SGEMM tile sizes
 int profile_sgemm_splitkpi(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
   int results = 0;
+  /*128x128x8*/
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 5 >(output, options, config, "sgemm_128x128x8_splitk_pi_split5", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 8 >(output, options, config, "sgemm_128x128x8_splitk_pi_split8", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "sgemm_128x128x8_splitk_pi_split10", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 16 >(output, options, config, "sgemm_128x128x8_splitk_pi_split16", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 20 >(output, options, config, "sgemm_128x128x8_splitk_pi_split20", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 24 >(output, options, config, "sgemm_128x128x8_splitk_pi_split24", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 28 >(output, options, config, "sgemm_128x128x8_splitk_pi_split28", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 32 >(output, options, config, "sgemm_128x128x8_splitk_pi_split32", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 40 >(output, options, config, "sgemm_128x128x8_splitk_pi_split40", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 48 >(output, options, config, "sgemm_128x128x8_splitk_pi_split48", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 56 >(output, options, config, "sgemm_128x128x8_splitk_pi_split56", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 64 >(output, options, config, "sgemm_128x128x8_splitk_pi_split64", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 72 >(output, options, config, "sgemm_128x128x8_splitk_pi_split72", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 80 >(output, options, config, "sgemm_128x128x8_splitk_pi_split80", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 88 >(output, options, config, "sgemm_128x128x8_splitk_pi_split88", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 96 >(output, options, config, "sgemm_128x128x8_splitk_pi_split96", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 104 >(output, options, config, "sgemm_128x128x8_splitk_pi_split104", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 112 >(output, options, config, "sgemm_128x128x8_splitk_pi_split112", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 120 >(output, options, config, "sgemm_128x128x8_splitk_pi_split120", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 128 >(output, options, config, "sgemm_128x128x8_splitk_pi_split128", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 136 >(output, options, config, "sgemm_128x128x8_splitk_pi_split136", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 144 >(output, options, config, "sgemm_128x128x8_splitk_pi_split144", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 152 >(output, options, config, "sgemm_128x128x8_splitk_pi_split152", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 160 >(output, options, config, "sgemm_128x128x8_splitk_pi_split160", "128x128");
 
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, 32 >(output, options, config, "sgemm_128x128x8_splitk_pi_split32", "128x128");
-
+#ifdef EXHAUSTIVE_PROF
   /*128x64x8*/
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, 8 >(output, options, config, "sgemm_128x64x8_splitk_pi_split8", "128x64");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, 16 >(output, options, config, "sgemm_128x64x8_splitk_pi_split16", "128x64");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, 20 >(output, options, config, "sgemm_128x64x8_splitk_pi_split20", "128x64");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, 24 >(output, options, config, "sgemm_128x64x8_splitk_pi_split24", "128x64");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, 28 >(output, options, config, "sgemm_128x64x8_splitk_pi_split28", "128x64");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, 32 >(output, options, config, "sgemm_128x64x8_splitk_pi_split32", "128x64");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, 64 >(output, options, config, "sgemm_128x64x8_splitk_pi_split64", "128x64");
-  /*128x32x8*/
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, 8 >(output, options, config, "sgemm_128x32x8_splitk_pi_split8", "128x32");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, 16 >(output, options, config, "sgemm_128x32x8_splitk_pi_split16", "128x32");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, 20 >(output, options, config, "sgemm_128x32x8_splitk_pi_split20", "128x32");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, 24 >(output, options, config, "sgemm_128x32x8_splitk_pi_split24", "128x32");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, 28 >(output, options, config, "sgemm_128x32x8_splitk_pi_split28", "128x32");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, 32 >(output, options, config, "sgemm_128x32x8_splitk_pi_split32", "128x32");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, 64 >(output, options, config, "sgemm_128x32x8_splitk_pi_split64", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 5 >(output, options, config, "sgemm_128x64x8_splitk_pi_split5", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 8 >(output, options, config, "sgemm_128x64x8_splitk_pi_split8", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "sgemm_128x64x8_splitk_pi_split10", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 16 >(output, options, config, "sgemm_128x64x8_splitk_pi_split16", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 20 >(output, options, config, "sgemm_128x64x8_splitk_pi_split20", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 24 >(output, options, config, "sgemm_128x64x8_splitk_pi_split24", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 28 >(output, options, config, "sgemm_128x64x8_splitk_pi_split28", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 32 >(output, options, config, "sgemm_128x64x8_splitk_pi_split32", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 40 >(output, options, config, "sgemm_128x64x8_splitk_pi_split40", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 48 >(output, options, config, "sgemm_128x64x8_splitk_pi_split48", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 56 >(output, options, config, "sgemm_128x64x8_splitk_pi_split56", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 64 >(output, options, config, "sgemm_128x64x8_splitk_pi_split64", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 72 >(output, options, config, "sgemm_128x64x8_splitk_pi_split72", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 80 >(output, options, config, "sgemm_128x64x8_splitk_pi_split80", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 88 >(output, options, config, "sgemm_128x64x8_splitk_pi_split88", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 96 >(output, options, config, "sgemm_128x64x8_splitk_pi_split96", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 104 >(output, options, config, "sgemm_128x64x8_splitk_pi_split104", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 112 >(output, options, config, "sgemm_128x64x8_splitk_pi_split112", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 120 >(output, options, config, "sgemm_128x64x8_splitk_pi_split120", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 128 >(output, options, config, "sgemm_128x64x8_splitk_pi_split128", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 136 >(output, options, config, "sgemm_128x64x8_splitk_pi_split136", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 144 >(output, options, config, "sgemm_128x64x8_splitk_pi_split144", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 152 >(output, options, config, "sgemm_128x64x8_splitk_pi_split152", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 160 >(output, options, config, "sgemm_128x64x8_splitk_pi_split160", "128x64");
+
+  /*128x32x8*/ 
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 5  >(output, options, config, "sgemm_128x32x8_splitk_pi_split5", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 8  >(output, options, config, "sgemm_128x32x8_splitk_pi_split8", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "sgemm_128x32x8_splitk_pi_split10", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 16 >(output, options, config, "sgemm_128x32x8_splitk_pi_split16", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 20 >(output, options, config, "sgemm_128x32x8_splitk_pi_split20", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 24 >(output, options, config, "sgemm_128x32x8_splitk_pi_split24", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 28 >(output, options, config, "sgemm_128x32x8_splitk_pi_split28", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 32 >(output, options, config, "sgemm_128x32x8_splitk_pi_split32", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 40 >(output, options, config, "sgemm_128x32x8_splitk_pi_split40", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 48 >(output, options, config, "sgemm_128x32x8_splitk_pi_split48", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 56 >(output, options, config, "sgemm_128x32x8_splitk_pi_split56", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 64 >(output, options, config, "sgemm_128x32x8_splitk_pi_split64", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 72 >(output, options, config, "sgemm_128x32x8_splitk_pi_split72", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 80 >(output, options, config, "sgemm_128x32x8_splitk_pi_split80", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 88 >(output, options, config, "sgemm_128x32x8_splitk_pi_split88", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 96 >(output, options, config, "sgemm_128x32x8_splitk_pi_split96", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 104 >(output, options, config, "sgemm_128x32x8_splitk_pi_split104", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 112 >(output, options, config, "sgemm_128x32x8_splitk_pi_split112", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 120 >(output, options, config, "sgemm_128x32x8_splitk_pi_split120", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 128 >(output, options, config, "sgemm_128x32x8_splitk_pi_split128", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 136 >(output, options, config, "sgemm_128x32x8_splitk_pi_split136", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 144 >(output, options, config, "sgemm_128x32x8_splitk_pi_split144", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 152 >(output, options, config, "sgemm_128x32x8_splitk_pi_split152", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 160 >(output, options, config, "sgemm_128x32x8_splitk_pi_split160", "128x32");
+  
+  /*64x128*/
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 5 >(output, options, config, "sgemm_64x128x8_splitk_pi_split5", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 8 >(output, options, config, "sgemm_64x128x8_splitk_pi_split8", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "sgemm_64x128x8_splitk_pi_split10", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 16 >(output, options, config, "sgemm_64x128x8_splitk_pi_split16", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 20 >(output, options, config, "sgemm_64x128x8_splitk_pi_split20", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 24 >(output, options, config, "sgemm_64x128x8_splitk_pi_split24", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 28 >(output, options, config, "sgemm_64x128x8_splitk_pi_split28", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 32 >(output, options, config, "sgemm_64x128x8_splitk_pi_split32", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 40 >(output, options, config, "sgemm_64x128x8_splitk_pi_split40", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 48 >(output, options, config, "sgemm_64x128x8_splitk_pi_split48", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 56 >(output, options, config, "sgemm_64x128x8_splitk_pi_split56", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 64 >(output, options, config, "sgemm_64x128x8_splitk_pi_split64", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 72 >(output, options, config, "sgemm_64x128x8_splitk_pi_split72", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 80 >(output, options, config, "sgemm_64x128x8_splitk_pi_split80", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 88 >(output, options, config, "sgemm_64x128x8_splitk_pi_split88", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 96 >(output, options, config, "sgemm_64x128x8_splitk_pi_split96", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 104 >(output, options, config, "sgemm_64x128x8_splitk_pi_split104", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 112 >(output, options, config, "sgemm_64x128x8_splitk_pi_split112", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 120 >(output, options, config, "sgemm_64x128x8_splitk_pi_split120", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 128 >(output, options, config, "sgemm_64x128x8_splitk_pi_split128", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 136 >(output, options, config, "sgemm_64x128x8_splitk_pi_split136", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 144 >(output, options, config, "sgemm_64x128x8_splitk_pi_split144", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 152 >(output, options, config, "sgemm_64x128x8_splitk_pi_split152", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 160 >(output, options, config, "sgemm_64x128x8_splitk_pi_split160", "64x128");
+  
+  /*32x128*/
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 5 >(output, options, config, "sgemm_32x128x8_splitk_pi_split5", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 8 >(output, options, config, "sgemm_32x128x8_splitk_pi_split8", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "sgemm_32x128x8_splitk_pi_split10", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 16 >(output, options, config, "sgemm_32x128x8_splitk_pi_split16", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 20 >(output, options, config, "sgemm_32x128x8_splitk_pi_split20", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 24 >(output, options, config, "sgemm_32x128x8_splitk_pi_split24", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 28 >(output, options, config, "sgemm_32x128x8_splitk_pi_split28", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 32 >(output, options, config, "sgemm_32x128x8_splitk_pi_split32", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 40 >(output, options, config, "sgemm_32x128x8_splitk_pi_split40", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 48 >(output, options, config, "sgemm_32x128x8_splitk_pi_split48", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 56 >(output, options, config, "sgemm_32x128x8_splitk_pi_split56", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 64 >(output, options, config, "sgemm_32x128x8_splitk_pi_split64", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 72 >(output, options, config, "sgemm_32x128x8_splitk_pi_split72", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 80 >(output, options, config, "sgemm_32x128x8_splitk_pi_split80", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 88 >(output, options, config, "sgemm_32x128x8_splitk_pi_split88", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 96 >(output, options, config, "sgemm_32x128x8_splitk_pi_split96", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 104 >(output, options, config, "sgemm_32x128x8_splitk_pi_split104", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 112 >(output, options, config, "sgemm_32x128x8_splitk_pi_split112", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 120 >(output, options, config, "sgemm_32x128x8_splitk_pi_split120", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 128 >(output, options, config, "sgemm_32x128x8_splitk_pi_split128", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 136 >(output, options, config, "sgemm_32x128x8_splitk_pi_split136", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 144 >(output, options, config, "sgemm_32x128x8_splitk_pi_split144", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 152 >(output, options, config, "sgemm_32x128x8_splitk_pi_split152", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 160 >(output, options, config, "sgemm_32x128x8_splitk_pi_split160", "32x128");
+
+  /*64x64*/
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 5 >(output, options, config, "sgemm_64x64x8_splitk_pi_split5", "64x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 8 >(output, options, config, "sgemm_64x64x8_splitk_pi_split8", "64x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "sgemm_64x64x8_splitk_pi_split10", "64x64");
+
+#endif //#ifdef EXHAUSTIVE_PROF
 
   return results;
 }

tools/test/perf/gemm/volta884_gemm.cu

@@ -0,0 +1,183 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#include "tools/test/perf/gemm/cutlass_volta884_dispatch.h"
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace perf {
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename OutputTile, typename AccumHostType>
+int profile_volta884_gemm_kernel(
+  TestbenchOutput<GemmProblem> &output,
+  TestbenchOptions const &options,
+  Config const &config,
+  std::string const &name,
+  std::string const &algo) {
+
+  int results = 0;
+
+  // compute capability check
+  if (!options.compute_capability(7, 0)) {
+    return 0;
+  }
+
+  typedef typename cutlass::TypeTraits<AccumHostType>::device_type AccumDevType;
+
+#if CUTLASS_ENABLE_TENSOR_CORE_MMA
+  typedef perf::GemmProfiler<
+    cutlass::half_t,
+    cutlass::half_t,
+    AccumHostType,
+    AccumHostType,
+    AccumHostType> GemmProfiler;
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_nn", options, config, algo);
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_nt", options, config, algo);
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_tn", options, config, algo);
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      cutlass::gemm::IdentityBlockSwizzle,
+      true
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_tt", options, config, algo);
+  }
+
+  #endif // if defined(CUTLASS_ENABLE_TENSOR_CORE_MMA)
+
+  return results;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+int profile_volta884_gemm(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
+  int results = 0;
+
+  
+  
+
+  results |= profile_volta884_gemm_kernel<cutlass::Shape<32, 128, 128>, float >(output, options, config, "s884gemm", "128x128");
+
+  results |= profile_volta884_gemm_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t >(output, options, config, "h884gemm", "128x128");
+
+#ifdef EXHAUSTIVE_PROF
+  results |= profile_volta884_gemm_kernel<cutlass::Shape<32, 128, 256>, float >(output, options, config, "s884gemm_256x128", "256x128");
+
+  results |= profile_volta884_gemm_kernel<cutlass::Shape<32, 64, 128>, float >(output, options, config, "s884gemm_128x64", "128x64");
+
+  results |= profile_volta884_gemm_kernel<cutlass::Shape<32, 64, 64>, float >(output, options, config, "s884gemm_64x64", "64x64");
+
+  results |= profile_volta884_gemm_kernel<cutlass::Shape<32, 128, 256>, cutlass::half_t >(output, options, config, "h884gemm_256x128", "256x128");
+
+  results |= profile_volta884_gemm_kernel<cutlass::Shape<32, 64, 128>, cutlass::half_t >(output, options, config, "h884gemm_128x64", "128x64");
+
+  results |= profile_volta884_gemm_kernel<cutlass::Shape<32, 64, 64>, cutlass::half_t >(output, options, config, "h884gemm_64x64", "64x64");
+#endif // defined EXHAUSTIVE_PROF
+
+  return results;
+}
+
+struct Volta884GemmRegistrar {
+  Volta884GemmRegistrar() { RegisterGemmProfileFunc(profile_volta884_gemm); }
+};
+
+volatile Volta884GemmRegistrar _Volta884GemmRegistrar;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace perf
+
+////////////////////////////////////////////////////////////////////////////////////////////////////

tools/test/perf/gemm/volta884_gemm_cta_rasterization_nn.cu

@@ -0,0 +1,242 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#include "tools/test/perf/gemm/cutlass_volta884_dispatch.h"
+
+#ifdef EXHAUSTIVE_PROF
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace perf {
+template <typename OutputTile, typename AccumHostType>
+int profile_volta884_gemm_cta_rasterization_nn_kernel(
+  TestbenchOutput<GemmProblem> &output,
+  TestbenchOptions const &options,
+  Config const &config,
+  std::string const &name,
+  std::string const &algo) {
+
+  int results = 0;
+
+  // compute capability check
+  if (!options.compute_capability(7, 0)) {
+    return 0;
+  }
+
+  typedef typename cutlass::TypeTraits<AccumHostType>::device_type AccumDevType;
+
+#if CUTLASS_ENABLE_TENSOR_CORE_MMA
+  typedef perf::GemmProfiler<
+    cutlass::half_t,
+    cutlass::half_t,
+    AccumHostType,
+    AccumHostType,
+    AccumHostType> GemmProfiler;
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_1_one_nn", options, config, algo + "_row_1_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_1_B_nn", options, config, algo + "_row_1_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_2_one_nn", options, config, algo + "_row_2_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_2_B_nn", options, config, algo + "_row_2_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_1_one_nn", options, config, algo + "_col_1_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_1_B_nn", options, config, algo + "_col_1_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_2_one_nn", options, config, algo + "_col_2_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_2_B_nn", options, config, algo + "_col_2_B");
+  }
+#endif // if defined(CUTLASS_ENABLE_TENSOR_CORE_MMA)
+
+  return results;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+int profile_volta884_gemm_cta_rasterization_nn(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
+  int results = 0;
+
+  results |= profile_volta884_gemm_cta_rasterization_nn_kernel<cutlass::Shape<32, 128, 128>, float >(output, options, config, "s884gemm", "128x128");
+
+  results |= profile_volta884_gemm_cta_rasterization_nn_kernel<cutlass::Shape<32, 128, 256>, float >(output, options, config, "s884gemm_256x128", "256x128");
+
+  return results;
+}
+
+struct Volta884GemmCTARasterizationNNRegistrar {
+  Volta884GemmCTARasterizationNNRegistrar() { RegisterGemmProfileFunc(profile_volta884_gemm_cta_rasterization_nn); }
+};
+
+volatile Volta884GemmCTARasterizationNNRegistrar _Volta884CTARasterizationNNGemmRegistrar;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace perf
+#endif // if defined(EXHAUSTIVE_PROF)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////

tools/test/perf/gemm/volta884_gemm_cta_rasterization_nt.cu

@@ -0,0 +1,247 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#include "tools/test/perf/gemm/cutlass_volta884_dispatch.h"
+
+#ifdef EXHAUSTIVE_PROF
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace perf {
+template <typename OutputTile, typename AccumHostType>
+int profile_volta884_gemm_cta_rasterization_nt_kernel(
+  TestbenchOutput<GemmProblem> &output,
+  TestbenchOptions const &options,
+  Config const &config,
+  std::string const &name,
+  std::string const &algo) {
+
+  int results = 0;
+
+  // compute capability check
+  if (!options.compute_capability(7, 0)) {
+    return 0;
+  }
+
+  typedef typename cutlass::TypeTraits<AccumHostType>::device_type AccumDevType;
+
+#if CUTLASS_ENABLE_TENSOR_CORE_MMA
+  typedef perf::GemmProfiler<
+    cutlass::half_t,
+    cutlass::half_t,
+    AccumHostType,
+    AccumHostType,
+    AccumHostType> GemmProfiler;
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_1_one_nt", options, config, algo + "_row_1_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_1_B_nt", options, config, algo + "_row_1_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_2_one_nt", options, config, algo + "_row_2_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_2_B_nt", options, config, algo + "_row_2_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_1_one_nt", options, config, algo + "_col_1_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_1_B_nt", options, config, algo + "_col_1_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_2_one_nt", options, config, algo + "_col_2_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_2_B_nt", options, config, algo + "_col_2_B");
+  }
+#endif // if defined(CUTLASS_ENABLE_TENSOR_CORE_MMA)
+
+  return results;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+int profile_volta884_gemm_cta_rasterization_nt(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
+  int results = 0;
+
+  results |= profile_volta884_gemm_cta_rasterization_nt_kernel<cutlass::Shape<32, 128, 128>, float >(output, options, config, "s884gemm", "128x128");
+
+  results |= profile_volta884_gemm_cta_rasterization_nt_kernel<cutlass::Shape<32, 128, 256>, float >(output, options, config, "s884gemm_256x128", "256x128");
+
+  results |= profile_volta884_gemm_cta_rasterization_nt_kernel<cutlass::Shape<32, 64, 128>, float >(output, options, config, "s884gemm_128x64", "128x64");
+
+  results |= profile_volta884_gemm_cta_rasterization_nt_kernel<cutlass::Shape<32, 64, 64>, float >(output, options, config, "s884gemm_64x64", "64x64");
+
+  return results;
+}
+
+struct Volta884GemmCTARasterizationNTRegistrar {
+  Volta884GemmCTARasterizationNTRegistrar() { RegisterGemmProfileFunc(profile_volta884_gemm_cta_rasterization_nt); }
+};
+
+volatile Volta884GemmCTARasterizationNTRegistrar _Volta884CTARasterizationNTGemmRegistrar;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace perf
+
+#endif // if defined(EXHAUSTIVE_PROF)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////

tools/test/perf/gemm/volta884_gemm_cta_rasterization_tn.cu

@@ -0,0 +1,247 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#include "tools/test/perf/gemm/cutlass_volta884_dispatch.h"
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace perf {
+template <typename OutputTile, typename AccumHostType>
+int profile_volta884_gemm_cta_rasterization_tn_kernel(
+  TestbenchOutput<GemmProblem> &output,
+  TestbenchOptions const &options,
+  Config const &config,
+  std::string const &name,
+  std::string const &algo) {
+
+  int results = 0;
+
+  // compute capability check
+  if (!options.compute_capability(7, 0)) {
+    return 0;
+  }
+
+  typedef typename cutlass::TypeTraits<AccumHostType>::device_type AccumDevType;
+
+#if CUTLASS_ENABLE_TENSOR_CORE_MMA
+  typedef perf::GemmProfiler<
+    cutlass::half_t,
+    cutlass::half_t,
+    AccumHostType,
+    AccumHostType,
+    AccumHostType> GemmProfiler;
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_1_one_tn", options, config, algo + "_row_1_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_1_B_tn", options, config, algo + "_row_1_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_2_one_tn", options, config, algo + "_row_2_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_2_B_tn", options, config, algo + "_row_2_B");
+  }
+
+#ifdef EXHAUSTIVE_PROF
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_1_one_tn", options, config, algo + "_col_1_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_1_B_tn", options, config, algo + "_col_1_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_2_one_tn", options, config, algo + "_col_2_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_2_B_tn", options, config, algo + "_col_2_B");
+  }
+#endif // if defined(EXHAUSTIVE_PROF)
+#endif // if defined(CUTLASS_ENABLE_TENSOR_CORE_MMA)
+
+  return results;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+int profile_volta884_gemm_cta_rasterization_tn(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
+  int results = 0;
+
+  results |= profile_volta884_gemm_cta_rasterization_tn_kernel<cutlass::Shape<32, 128, 128>, float >(output, options, config, "s884gemm", "128x128");
+
+#ifdef EXHAUSTIVE_PROF
+  results |= profile_volta884_gemm_cta_rasterization_tn_kernel<cutlass::Shape<32, 128, 256>, float >(output, options, config, "s884gemm_256x128", "256x128");
+
+  results |= profile_volta884_gemm_cta_rasterization_tn_kernel<cutlass::Shape<32, 64, 128>, float >(output, options, config, "s884gemm_128x64", "128x64");
+
+  results |= profile_volta884_gemm_cta_rasterization_tn_kernel<cutlass::Shape<32, 64, 64>, float >(output, options, config, "s884gemm_64x64", "64x64");
+#endif // if defined( EXHAUSTIVE_PROF)
+
+  return results;
+}
+
+struct Volta884GemmCTARasterizationTNRegistrar {
+  Volta884GemmCTARasterizationTNRegistrar() { RegisterGemmProfileFunc(profile_volta884_gemm_cta_rasterization_tn); }
+};
+
+volatile Volta884GemmCTARasterizationTNRegistrar _Volta884CTARasterizationTNGemmRegistrar;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace perf
+
+////////////////////////////////////////////////////////////////////////////////////////////////////

tools/test/perf/gemm/volta884_gemm_cta_rasterization_tt.cu

@@ -0,0 +1,255 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#include "tools/test/perf/gemm/cutlass_volta884_dispatch.h"
+
+#ifdef EXHAUSTIVE_PROF
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace perf {
+template <typename OutputTile, typename AccumHostType>
+int profile_volta884_gemm_cta_rasterization_tt_kernel(
+  TestbenchOutput<GemmProblem> &output,
+  TestbenchOptions const &options,
+  Config const &config,
+  std::string const &name,
+  std::string const &algo) {
+
+  int results = 0;
+
+  // compute capability check
+  if (!options.compute_capability(7, 0)) {
+    return 0;
+  }
+
+  typedef typename cutlass::TypeTraits<AccumHostType>::device_type AccumDevType;
+
+#if CUTLASS_ENABLE_TENSOR_CORE_MMA
+  typedef perf::GemmProfiler<
+    cutlass::half_t,
+    cutlass::half_t,
+    AccumHostType,
+    AccumHostType,
+    AccumHostType> GemmProfiler;
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>,
+      true
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_1_one_tt", options, config, algo + "_row_1_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>,
+      true
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_1_B_tt", options, config, algo + "_row_1_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>,
+      true
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_2_one_tt", options, config, algo + "_row_2_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>,
+      true
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_2_B_tt", options, config, algo + "_row_2_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>,
+      true
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_1_one_tt", options, config, algo + "_col_1_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>,
+      true
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_1_B_tt", options, config, algo + "_col_1_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>,
+      true
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_2_one_tt", options, config, algo + "_col_2_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>,
+      true
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_2_B_tt", options, config, algo + "_col_2_B");
+  }
+#endif // if defined(CUTLASS_ENABLE_TENSOR_CORE_MMA)
+
+  return results;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+int profile_volta884_gemm_cta_rasterization_tt(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
+  int results = 0;
+
+  results |= profile_volta884_gemm_cta_rasterization_tt_kernel<cutlass::Shape<32, 128, 128>, float >(output, options, config, "s884gemm", "128x128");
+
+  results |= profile_volta884_gemm_cta_rasterization_tt_kernel<cutlass::Shape<32, 128, 256>, float >(output, options, config, "s884gemm_256x128", "256x128");
+
+  results |= profile_volta884_gemm_cta_rasterization_tt_kernel<cutlass::Shape<32, 64, 128>, float >(output, options, config, "s884gemm_128x64", "128x64");
+
+  results |= profile_volta884_gemm_cta_rasterization_tt_kernel<cutlass::Shape<32, 64, 64>, float >(output, options, config, "s884gemm_64x64", "64x64");
+
+  return results;
+}
+
+struct Volta884GemmCTARasterizationTTRegistrar {
+  Volta884GemmCTARasterizationTTRegistrar() { RegisterGemmProfileFunc(profile_volta884_gemm_cta_rasterization_tt); }
+};
+
+volatile Volta884GemmCTARasterizationTTRegistrar _Volta884CTARasterizationTTGemmRegistrar;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace perf
+
+#endif // if defined(EXHAUSTIVE_PROF)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////

tools/test/perf/gemm/volta884_gemm_splitK.cu

@@ -0,0 +1,324 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#include "tools/test/perf/gemm/cutlass_volta884_dispatch_splitK_PI.h"
+#include "cutlass/reduction/batched_reduction_traits.h"
+#include "cutlass/gemm/device_gemm_traits.h"
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace perf {
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename OutputTile, typename AccumHostType, typename threadReductionShape, int splits_count>
+int profile_volta884_gemm_splitkpi_kernel(
+  TestbenchOutput<GemmProblem> &output,
+  TestbenchOptions const &options,
+  Config const &config,
+  std::string const &name,
+  std::string const &algo) {
+
+  int results = 0;
+
+  // compute capability check
+  if (!options.compute_capability(7, 0)) {
+    return 0;
+  }
+
+  typedef typename cutlass::TypeTraits<AccumHostType>::device_type AccumDevType;
+
+#if CUTLASS_ENABLE_TENSOR_CORE_MMA
+  typedef perf::GemmProfiler<
+    cutlass::half_t,
+    cutlass::half_t,
+    cutlass::half_t,
+    AccumHostType,
+    AccumHostType> GemmProfiler;
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2
+    > GemmTraits;
+
+    /*batched reduction traits*/
+    typedef cutlass::reduction::BatchedReductionTraits<AccumDevType,
+      half,
+      half,
+      AccumDevType,
+      AccumDevType,
+      splits_count,
+      cutlass::Shape<1, 1, 128>,
+      cutlass::Shape<1, 1, 64>,
+      threadReductionShape >
+      BatchedReductionTraits;
+
+    // create a device gemm 
+    typedef typename cutlass::gemm::SplitkPIGemmTraits<GemmTraits, BatchedReductionTraits> deviceGemmTraits;
+    typedef Volta884GemmDispatchSplitKPI<deviceGemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_nn", options, config, algo);
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2
+    > GemmTraits;
+
+    /*batched reduction traits*/
+    typedef cutlass::reduction::BatchedReductionTraits<AccumDevType,
+      half,
+      half,
+      AccumDevType,
+      AccumDevType,
+      splits_count,
+      cutlass::Shape<1, 1, 128>,
+      cutlass::Shape<1, 1, 64>,
+      threadReductionShape >
+      BatchedReductionTraits;
+
+    // create a device gemm 
+    typedef typename cutlass::gemm::SplitkPIGemmTraits<GemmTraits, BatchedReductionTraits> deviceGemmTraits;
+    typedef Volta884GemmDispatchSplitKPI<deviceGemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_nt", options, config, algo);
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2
+    > GemmTraits;
+
+    /*batched reduction traits*/
+    typedef cutlass::reduction::BatchedReductionTraits<AccumDevType,
+      half,
+      half,
+      AccumDevType,
+      AccumDevType,
+      splits_count,
+      cutlass::Shape<1, 1, 128>,
+      cutlass::Shape<1, 1, 64>,
+      threadReductionShape >
+      BatchedReductionTraits;
+
+    // create a device gemm 
+    typedef typename cutlass::gemm::SplitkPIGemmTraits<GemmTraits, BatchedReductionTraits> deviceGemmTraits;
+    typedef Volta884GemmDispatchSplitKPI<deviceGemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_tn", options, config, algo);
+}
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2
+    > GemmTraits;
+
+    /*batched reduction traits*/
+    typedef cutlass::reduction::BatchedReductionTraits<AccumDevType,
+      half,
+      half,
+      AccumDevType,
+      AccumDevType,
+      splits_count,
+      cutlass::Shape<1, 1, 128>,
+      cutlass::Shape<1, 1, 64>,
+      threadReductionShape >
+      BatchedReductionTraits;
+
+    // create a device gemm 
+    typedef typename cutlass::gemm::SplitkPIGemmTraits<GemmTraits, BatchedReductionTraits> deviceGemmTraits;
+    typedef Volta884GemmDispatchSplitKPI<deviceGemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_tt", options, config, algo);
+  }
+
+  #endif // if defined(CUTLASS_ENABLE_TENSOR_CORE_MMA)
+
+  return results;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+int profile_volta884_gemm_splitkpi(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
+  int results = 0;
+
+  //results |= profile_volta884_gemm_kernel<cutlass::Shape<32, 128, 128>, float >(output, options, config, "s884gemm", "128x128");
+
+  // half accum
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 5 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits5", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 8 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits8", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits10", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 16 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits16", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 20 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits20", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 24 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits24", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 28 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits28", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 32 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits32", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 40 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits40", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 48 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits48", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 56 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits56", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 64 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits64", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 72 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits72", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 80 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits80", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 88 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits88", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 96 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits96", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 104 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits104", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 112 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits112", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 120 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits120", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 128 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits128", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 136 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits136", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 144 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits144", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 152 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits152", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 160 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits160", "128x128");
+
+  // float accum
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 5 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits5", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 8 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits8", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits10", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 16 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits16", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 20 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits20", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 24 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits24", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 28 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits28", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 32 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits32", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 40 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits40", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 48 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits48", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 56 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits56", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 64 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits64", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 72 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits72", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 80 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits80", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 88 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits88", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 96 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits96", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 104 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits104", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 112 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits112", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 120 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits120", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 128 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits128", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 136 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits136", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 144 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits144", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 152 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits152", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 160 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits160", "128x128");
+
+#ifdef EXHAUSTIVE_PROF
+  // float accum 128x64
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 5 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits5", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 8 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits8", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits10", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 16 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits16", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 20 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits20", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 24 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits24", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 28 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits28", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 32 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits32", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 40 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits40", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 48 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits48", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 56 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits56", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 64 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits64", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 72 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits72", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 80 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits80", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 88 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits88", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 96 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits96", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 104 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits104", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 112 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits112", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 120 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits120", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 128 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits128", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 136 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits136", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 144 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits144", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 152 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits152", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 160 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits160", "128x64");
+
+  // float accum 64x128
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 5 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits5", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 8 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits8", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits10", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 16 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits16", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 20 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits20", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 24 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits24", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 28 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits28", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 32 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits32", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 40 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits40", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 48 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits48", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 56 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits56", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 64 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits64", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 72 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits72", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 80 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits80", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 88 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits88", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 96 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits96", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 104 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits104", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 112 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits112", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 120 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits120", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 128 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits128", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 136 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits136", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 144 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits144", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 152 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits152", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 160 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits160", "64x128");
+#endif //#ifdef EXHAUSTIVE_PROF
+  return results;
+}
+
+struct Volta884GemmSplitKPIRegistrar {
+  Volta884GemmSplitKPIRegistrar() { RegisterGemmProfileFunc(profile_volta884_gemm_splitkpi); }
+};
+
+volatile Volta884GemmSplitKPIRegistrar _Volta884GemmSplitKPIRegistrar;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace perf
+
+////////////////////////////////////////////////////////////////////////////////////////////////////

tools/test/perf/gemm/wmma_binary_gemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -30,7 +30,7 @@
 #include "cutlass/wmma_matrix.h"
 #ifdef CUTLASS_USE_WMMA_API
 #ifdef CUTLASS_USE_SUBBYTE_WMMA
-
+#pragma warning( disable : 4503)
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 #include "cutlass/gemm/gemm.h"

tools/test/perf/gemm/wmma_gemm.cu

@@ -1,27 +1,27 @@
 /***************************************************************************************************
-* Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
-*
-* Redistribution and use in source and binary forms, with or without modification, are permitted
-* provided that the following conditions are met:
-*     * Redistributions of source code must retain the above copyright notice, this list of
-*       conditions and the following disclaimer.
-*     * Redistributions in binary form must reproduce the above copyright notice, this list of
-*       conditions and the following disclaimer in the documentation and/or other materials
-*       provided with the distribution.
-*     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
-*       to endorse or promote products derived from this software without specific prior written
-*       permission.
-*
-* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
-* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
-* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
-* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
-* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*
-**************************************************************************************************/
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
 
 #include "cutlass/wmma_matrix.h"
 #ifdef CUTLASS_USE_WMMA_API
@@ -49,7 +49,11 @@ struct WmmaGemmDispatch {
   /// Indicate warp-level GEMM
   static bool const kThreadMultiplyAdd = false;
 
+  #if CUTLASS_ENABLE_CUBLAS
   static bool const kRunCuBLAS = true;
+  #else
+  static bool const kRunCuBLAS = false;
+  #endif
 
   static cutlass::MatrixLayout::Kind const kLayoutA = Traits::kLayoutA;
   static cutlass::MatrixLayout::Kind const kLayoutB = Traits::kLayoutB;
@@ -138,8 +142,8 @@ int profile_wmma_gemm_f32(TestbenchOutput<GemmProblem> &output, TestbenchOptions
 
   {
     typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
-      cutlass::MatrixLayout::kRowMajor>
-      WmmaGemmTraits;
+                                        cutlass::MatrixLayout::kRowMajor>
+    WmmaGemmTraits;
 
     typedef WmmaGemmDispatch<WmmaGemmTraits> Dispatch;
 
@@ -148,8 +152,8 @@ int profile_wmma_gemm_f32(TestbenchOutput<GemmProblem> &output, TestbenchOptions
 
   {
     typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
-      cutlass::MatrixLayout::kColumnMajor>
-      WmmaGemmTraits;
+                                        cutlass::MatrixLayout::kColumnMajor>
+    WmmaGemmTraits;
 
     typedef WmmaGemmDispatch<WmmaGemmTraits> Dispatch;
 
@@ -158,7 +162,7 @@ int profile_wmma_gemm_f32(TestbenchOutput<GemmProblem> &output, TestbenchOptions
 
   {
     typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
-      cutlass::MatrixLayout::kColumnMajor>
+                                        cutlass::MatrixLayout::kColumnMajor>
       WmmaGemmTraits;
 
     typedef WmmaGemmDispatch<WmmaGemmTraits> Dispatch;
@@ -168,7 +172,7 @@ int profile_wmma_gemm_f32(TestbenchOutput<GemmProblem> &output, TestbenchOptions
 
   {
     typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
-      cutlass::MatrixLayout::kRowMajor>
+                                        cutlass::MatrixLayout::kRowMajor>
       WmmaGemmTraits;
 
     typedef WmmaGemmDispatch<WmmaGemmTraits> Dispatch;
@@ -183,9 +187,9 @@ int profile_wmma_gemm_f32(TestbenchOutput<GemmProblem> &output, TestbenchOptions
 
 template <typename DummyT>
 int profile_wmma_gemm_f16(
-  TestbenchOutput<GemmProblem> &output,
-  TestbenchOptions const &options,
-  Config const &config) {
+    TestbenchOutput<GemmProblem> &output,
+    TestbenchOptions const &options,
+    Config const &config) {
 
   typedef perf::GemmProfiler<
     cutlass::half_t,
@@ -278,9 +282,9 @@ int profile_wmma_gemm_f16(
 
 template <typename DummyT>
 int profile_wmma_4_gemm_f16(
-  TestbenchOutput<GemmProblem> &output,
-  TestbenchOptions const &options,
-  Config const &config) {
+    TestbenchOutput<GemmProblem> &output,
+    TestbenchOptions const &options,
+    Config const &config) {
 
   typedef perf::GemmProfiler<
     cutlass::half_t,
@@ -547,11 +551,11 @@ struct WmmaGemmRegistrar {
     RegisterGemmProfileFunc(profile_wmma_gemm_f32<void>);
     RegisterGemmProfileFunc(profile_wmma_gemm_f16<void>);
 
-    //#ifdef EXHAUSTIVE_PROF
+//#ifdef EXHAUSTIVE_PROF
     RegisterGemmProfileFunc(profile_wmma_4_gemm_f16<void>);
     //fp32 accum with fp16 input and output
     RegisterGemmProfileFunc(profile_wmma_4_fp16_sgemm_fp16<void>);
-    //#endif  // defined EXHAUSTIVE_PROF
+//#endif  // defined EXHAUSTIVE_PROF
   }
 };
 
@@ -564,4 +568,3 @@ volatile WmmaGemmRegistrar _WmmaGemmRegistrar;
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 #endif  // defined CUTLASS_USE_WMMA_API
-

tools/test/perf/gemm/wmma_integer_gemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -29,8 +29,8 @@
 
 #include "cutlass/wmma_matrix.h"
 #ifdef CUTLASS_USE_WMMA_API
-#ifdef CUTLASS_USE_SUBBYTE_WMMA
-
+#ifdef CUTLASS_USE_INT_WMMA
+#pragma warning( disable : 4503)
 #include "cutlass/gemm/gemm.h"
 #include "cutlass/gemm/wmma_gemm_traits.h"
 #include "tools/test/perf/gemm/cutlass_dispatch.h"
@@ -92,6 +92,7 @@ struct WmmaIntegerGemmDispatch {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
+#ifdef CUTLASS_USE_SUBBYTE_WMMA
 template<typename Traits>
 struct WmmaIntegerGemmDispatch<Traits,
                                cutlass::Vector<cutlass::int4_t, 8>,
@@ -209,6 +210,7 @@ struct WmmaIntegerGemmDispatch<Traits,
   /// Launches kernel
   cudaError_t operator()() { return Gemm::launch(params); }
 };
+#endif //ifdef CUTLASS_USE_SUBBYTE_WMMA
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
@@ -221,7 +223,7 @@ int profile_wmma_integer_gemm(TestbenchOutput<GemmProblem> &output, TestbenchOpt
   int results = 0;
 
   // compute capability check
-  if (!options.compute_capability(7, 5)) {
+  if (!options.compute_capability(7, 2)) {
     return 0;
   }
 
@@ -398,6 +400,7 @@ int profile_wmma_integer_gemm(TestbenchOutput<GemmProblem> &output, TestbenchOpt
     return 0;
   }
 
+#ifdef CUTLASS_USE_SUBBYTE_WMMA
   {
     typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
                                           cutlass::MatrixLayout::kColumnMajor,
@@ -451,6 +454,7 @@ int profile_wmma_integer_gemm(TestbenchOutput<GemmProblem> &output, TestbenchOpt
 
     results |= profile_gemm<Dispatch, GemmProfiler>(output, "wmma_integer_gemm_u4_tn", options, config);
   }
+#endif //ifdef CUTLASS_USE_SUBBYTE_WMMA
 
   return results;
 }
@@ -461,7 +465,7 @@ int profile_wmma_integer_gemm(TestbenchOutput<GemmProblem> &output, TestbenchOpt
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-#else // ! CUTLASS_USE_SUBBYTE_WMMA
+#else // ! CUTLASS_USE_INT_WMMA
 
 namespace perf {
 

tools/test/perf/performance_result.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -28,7 +28,6 @@
 #include "cutlass/matrix_traits.h"
 #include "tools/util/command_line.h"
 #include "tools/test/perf/provider.h"
-
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 namespace perf {
@@ -175,7 +174,6 @@ inline std::ostream &operator<<(std::ostream &out, GemmProblem const &problem) {
 
   return out;
 }
-
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 /// Result object

tools/test/perf/provider.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/perf/testbench_configs.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -45,9 +45,9 @@ struct Config {
   // kernel to run
   std::vector<std::string> kernels;
 
-  /// Range of problem sizes
-  GemmProblemRange problem_range;
-
+  /// Range of problem sizes for GEMM
+  GemmProblemRange gemm_problem_range;
+  
   // Reference GFLOPs
   double gflops_ref;
 
@@ -121,8 +121,12 @@ struct TestbenchConfigs {
           if (item.compare("Kernel") == 0) kernel_idx = idx;
           if (item.compare("Beta") == 0) beta_idx = idx;
           if (item.compare("M") == 0) m_idx = idx;
-          if (item.compare("N") == 0) n_idx = idx;
-          if (item.compare("K") == 0) k_idx = idx;
+          if (item.compare("N") == 0) { 
+            n_idx = idx;
+          }
+          if (item.compare("K") == 0) { 
+            k_idx = idx; 
+          }
           if (item.compare("GFLOPs") == 0) gflops_idx = idx;
           if (item.compare("Runtime") == 0) runtime_idx = idx;
           if (item.compare("SOL") == 0) peak_throughput_idx = idx;
@@ -150,9 +154,9 @@ struct TestbenchConfigs {
           config.alpha = options.alpha;
           config.beta = strtod(tokens[beta_idx].c_str(), NULL);
           config.kernels.push_back(tokens[kernel_idx]);
-          config.problem_range.M = Range((int)strtol(tokens[m_idx].c_str(), NULL, 10));
-          config.problem_range.N = Range((int)strtol(tokens[n_idx].c_str(), NULL, 10));
-          config.problem_range.K = Range((int)strtol(tokens[k_idx].c_str(), NULL, 10));
+          config.gemm_problem_range.M = Range(tokens[m_idx]);
+          config.gemm_problem_range.N = Range(tokens[n_idx]);
+          config.gemm_problem_range.K = Range(tokens[k_idx]);
           config.gflops_ref = strtod(tokens[gflops_idx].c_str(), NULL);
           config.runtime_ref = strtod(tokens[runtime_idx].c_str(), NULL);
           config.peak_throughput_ref = strtod(tokens[peak_throughput_idx].c_str(), NULL);
@@ -172,7 +176,7 @@ struct TestbenchConfigs {
     for (int i = 0; i < options.kernels.size(); i++) {
       config.kernels.push_back(options.kernels[i]);
     }
-    config.problem_range = options.problem_range;
+    config.gemm_problem_range = options.gemm_problem_range;
     configs.push_back(config);
   }
 

tools/test/perf/testbench_options.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -66,6 +66,11 @@ struct Range {
   Range(int _start, int _end, int _increment = 1, Operator _op = Add)
       : start(_start), end(_end), increment(_increment), increment_op(_op) {}
 
+  Range(std::string _start) : increment(1), increment_op(Add) {
+    start = end = (int)strtol(_start.c_str(), NULL, 10);
+  }
+
+
   /// Returns the next item in series
   int next(int val) const {
     switch (increment_op) {
@@ -162,8 +167,6 @@ struct GemmProblemRange {
   }
 };
 
-////////////////////////////////////////////////////////////////////////////////////////////////////
-
 /// Defines a vector of string pairs
 typedef std::vector<std::pair<std::string, std::string> > KeyValueVector;
 
@@ -391,8 +394,8 @@ struct TestbenchOptions {
   /// Scalar value for GEMM
   double beta;
 
-  /// Range of problem sizes
-  GemmProblemRange problem_range;
+  /// Range of GEMM problem sizes
+  GemmProblemRange gemm_problem_range;
 
   /// If true, kernels are not executed, and no sleep waits are inserted
   bool dry_run;
@@ -418,7 +421,7 @@ struct TestbenchOptions {
       : initial_distribution(args),
         execution_mode(ExecutionMode::Profile),
         save_workspace_mode(WorkspaceSaveMode::Never),
-        problem_range(args),
+        gemm_problem_range(args),
         dry_run(false),
         sleep_time(1) {
 
@@ -473,6 +476,8 @@ struct TestbenchOptions {
         "igemm",
         "wmma_gemm",
         "wmma_gemm_f16",
+        "s884gemm",
+        "h884gemm",
         "wmma_binary_gemm",
         "wmma_integer_gemm",
         0
@@ -480,7 +485,8 @@ struct TestbenchOptions {
       char const *layouts[] = {"nn", "nt", "tn", "tt", 0};
       for (int i = 0; gemms[i]; ++i) {
         for (int j = 0; layouts[j]; ++j) {
-          if ((std::string(gemms[i]).compare("wmma_binary_gemm") == 0 ||
+          if ((
+               std::string(gemms[i]).compare("wmma_binary_gemm") == 0 ||
                std::string(gemms[i]).compare("wmma_integer_gemm") == 0)
                && std::string(layouts[j]).compare("tn") != 0) {
             continue;
@@ -488,7 +494,7 @@ struct TestbenchOptions {
           kernels.push_back(std::string(gemms[i]) + "_" + layouts[j]);
         }
       }
-
+      
     }
   }
 
@@ -596,15 +602,14 @@ struct TestbenchOptions {
         << "    Height of GEMM problem (number of rows of C). May specify a range with optional "
            "step size.\n"
 
-        << "  --n=<width>[:max width[:step]]                "
+        << "  --n=<width>[:max width[:step]] (GEMM-specific)"
         << "    Width of GEMM problem (number of columns of C). May specify a range with optional "
            "step size.\n"
 
-        << "  --k=<depth>[:max depth[:step]]                "
+        << "  --k=<depth>[:max depth[:step]] (GEMM-specific)"
         << "    Size of inner dimension of A and B. May specify a range with optional step size.\n"
-
         << "  --batch=<batch>                               "
-        << "    Number of batches for a bached gemm. "
+        << "    Number of batches for a batched gemm.\n"
 
         << "  --kernels=<{s|d|h|i|wmma_|wmma_binary_|wmma_integer_}gemm_{nn,nt,tn,tt}>\n"
         << "                                                "
@@ -641,13 +646,14 @@ struct TestbenchOptions {
     out << "\n\n"
         << "Example usage:\n\n"
 
-        << "# Runs one problem size for all kernels\n"
+        << "# Runs one problem size for all GEMM kernels\n"
         << "./tools/test/perf/cutlass_perf_test --m=10240 --n=1024 --k=1024\n\n"
 
         << "# Varies GEMM K dimension for SGEMM and IGEMM with column-major multiplicands\n"
         << "./tools/test/perf/cutlass_perf_test --m=10240 --n=4096 --k=1024:8192:128 "
            "--kernels=sgemm_nn,igemm_nn\n\n"
-
+        << " # Executes GEMM kernel on Volta Tensor Cores\n"
+        << " $ ./tools/test/perf/cutlass_perf_test --kernels=s884gemm_nt\n\n"
         << std::flush;
   }
 };

tools/test/perf/testbench_output.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification, are permitted
 # provided that the following conditions are met:
@@ -43,11 +43,6 @@ set(CUTLASS_UNIT_TEST_HEADERS
   reduction/test_batched_reduction.h
 )
 
-set(CUTLASS_UNIT_TEST_SOURCES_BACKUP
-  cutlass_unit_test.cpp
-  gemm/batched_strided_sgemm_128x128x8.cu
-)
-
 set(CUTLASS_UNIT_TEST_SOURCES
   cutlass_unit_test.cpp
   tile_iterator_test.cu
@@ -85,6 +80,7 @@ set(CUTLASS_UNIT_TEST_SOURCES
   gemm/sgemm_64x64x16.cu
   gemm/sgemm_64x32x8.cu
   gemm/sgemm_64x32x16.cu
+  gemm/sgemm_32x128x8.cu
   gemm/fp16_sgemm_fp32_128x128x16.cu
   gemm/fp16_sgemm_fp16_128x128x16.cu
   gemm/wmma_gemm.cu
@@ -102,7 +98,13 @@ set(CUTLASS_UNIT_TEST_SOURCES
   gemm/batched_strided_hgemm_128x128x8.cu
   gemm/batched_strided_wmma_gemm.cu
   gemm/batched_strided_fp16_wmma_gemm_fp16.cu
+  gemm/batched_strided_volta884_hgemm.cu
   gemm/epilogue_functor.cu
+  gemm/volta884_gemm_epilogue.cu
+  gemm/volta884_h884gemm.cu
+  gemm/volta884_gemm.cu
+  gemm/volta884_gemm_threadblock_swizzle.cu
+  gemm/volta884_h884gemm_threadblock_swizzle.cu
   reduction/batched_reduction.cu
   reduction/mixed_batched_reduction.cu
   gemm/splitK_sgemm.cu
@@ -111,7 +113,9 @@ set(CUTLASS_UNIT_TEST_SOURCES
   gemm/splitK_dgemm.cu
   gemm/splitK_hgemm.cu
   gemm/splitK_wmma_gemm.cu
+  gemm/splitK_volta884_hgemm.cu
   gemm/partitionedK_sgemm_128x128x8.cu
+  gemm/partitionedK_volta884_hgemm.cu
 )
 
 if (CUTLASS_NVRTC_ENABLE)
@@ -144,6 +148,7 @@ if (CUTLASS_NVRTC_ENABLE)
   endif()
 endif()
 
-target_link_libraries(cutlass_unit_test ${CUBLAS_LIBRARY})
-
+if(CUTLASS_ENABLE_CUBLAS)
+    target_link_libraries(cutlass_unit_test ${CUBLAS_LIBRARY})
+endif()
 

tools/test/unit/core/layout_verification.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/core/layout_verification.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/core/predicate_vector.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/core/tensor_ref.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/core/tensor_view.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/core/tile_iterator.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/core/zip_tile_iterator.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/cutlass_unit_test.cpp

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -84,9 +84,9 @@ void set_gtest_flag() {
     { "*wmma*",                     70, false },
     { "WmmaInt8*",                  72, false },
     { "*wmmaInt8*",                 72, false },
-    { "WmmaInt4*",                  75, true  },
+    { "WmmaInt4*",                  75, true },
     { "*wmmaInt4*",                 75, true },
-    { "WmmaBinary*",                75, true  },
+    { "WmmaBinary*",                75, true },
     { "*wmmaBinary*",               75, true },
     { 0, 0, false }
   };

tools/test/unit/cutlass_unit_test.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -29,3 +29,7 @@
 #include <gtest/gtest.h>
 #pragma diag_warning boolean_controlling_expr_is_constant
 #pragma warning( disable : 4503)
+
+#if !defined(CUTLASS_ENABLE_CUBLAS)
+#define CUTLASS_ENABLE_CUBLAS 0
+#endif

tools/test/unit/gemm/batched_strided_dgemm_128x128x8.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/batched_strided_fp16_wmma_gemm_fp16.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
-* Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+* Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification, are permitted
 * provided that the following conditions are met:

tools/test/unit/gemm/batched_strided_hgemm_128x128x8.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/batched_strided_sgemm_128x128x8.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/batched_strided_volta884_hgemm.cu

@@ -0,0 +1,114 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+#include <cublas_v2.h>
+#include <cstring>
+#include "cutlass_unit_test.h"
+
+#include "tools/util/half.h"
+#include "tools/util/host_tensor.h"
+#include "tools/util/tensor_view_io.h"
+
+#include "cutlass/gemm/volta884_gemm_traits.h"
+#include "cutlass/gemm/gemm.h"
+
+#include "tools/test/unit/gemm/gemm_testbed.h"
+#include "tools/test/unit/gemm/run_gemm.h"
+
+#if CUTLASS_ENABLE_TENSOR_CORE_MMA
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_h884gemm_batched_strided_64x64x32_nt, 64x128x64x3) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+
+  run_batched_strided_gemm<GemmTraits>(64, 128, 64, 3);
+}
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_h884gemm_batched_strided_64x64x32_nn, 64x128x64x3) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+
+  run_batched_strided_gemm<GemmTraits>(64, 128, 64, 3);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_h884gemm_batched_strided_64x64x32_tn, 64x128x64x3) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+
+  run_batched_strided_gemm<GemmTraits>(64, 128, 64, 3);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_h884gemm_batched_strided_64x64x32_tt, 64x128x64x3) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+
+  run_batched_strided_gemm<GemmTraits>(64, 128, 64, 3);
+}
+
+#endif // if defined(CUTLASS_ENABLE_TENSOR_CORE_MMA)

tools/test/unit/gemm/batched_strided_wmma_gemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
-* Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+* Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification, are permitted
 * provided that the following conditions are met:

tools/test/unit/gemm/dgemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/epilogue_functor.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -83,7 +83,7 @@ TEST(Sgemm_epilogue_functor, device_ptr_mode_sgemm_1024x512x128_nt) {
   //
   // Construct a CUTLASS GEMM and initialize parameters
   //
-  typedef typename SgemmTraits::KernelClass Gemm;
+  typedef cutlass::gemm::Gemm<SgemmTraits> Gemm;
   typename Gemm::Params params;
 
   params.initialize(testbed.M(),

tools/test/unit/gemm/fp16_sgemm_fp16_128x128x16.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/fp16_sgemm_fp32_128x128x16.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/fp16_wmma_gemm_fp16.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/fp16_wmma_gemm_fp16_non_multiple16.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
-* Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+* Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification, are permitted
 * provided that the following conditions are met:

tools/test/unit/gemm/gemm_nvrtc.h

@@ -88,8 +88,6 @@ static __host__ void run_gemm_nvrtc(
 
   std::string type_name;
 #if 0
-  // TODO Ideally we'd use nvrtcGetTypeName to determine the type, but it cannot resolve enum symbol names
-  // As altername solution we might want to implement to_string<GemmTraits>() to get the traits string.
   nvrtcGetTypeName<typename GemmTraits_>(&type_name);
 #else
   type_name = gemm_traits;

tools/test/unit/gemm/gemm_testbed.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -204,7 +204,7 @@ struct GemmTestbed {
   struct RandomBitGenerator {
     RandomBitGenerator(int seed = -1) { srand(seed); }
 
-    T operator()() {
+     T operator()() {
       uint32_t val = 0;
       for (int i = 0; i < 32; i++) {
         val |= rand() % 2;
@@ -268,7 +268,10 @@ struct GemmTestbed {
 
   /// partitionK count
   int partitionK_count;
-  
+
+  /// each partition should be mulitples of partitionK_multiple
+  int partitionK_multiple;
+
   /// distance between A[i] and A[i+1] for strided batched gemm
   long long int batch_stride_A;
 
@@ -316,13 +319,19 @@ struct GemmTestbed {
         algorithm(algorithm_),
         batch_count(1),
         partitionK_count(1),
+        partitionK_multiple(1),
         batch_stride_A(static_cast<long long int>(0)),
         batch_stride_B(static_cast<long long int>(0)),
         batch_stride_C(static_cast<long long int>(0)) {
+
+    #if CUTLASS_ENABLE_CUBLAS
     status = cublasCreate(&handle);
     if (status != CUBLAS_STATUS_SUCCESS) {
       throw cutlass::cuda_exception("Failed to create CUBLAS handle");
     }
+    #else
+    status = CUBLAS_STATUS_NOT_INITIALIZED;
+    #endif
 
     resize(A, M_, K_, layout_a);
     resize(B, K_, N_, layout_b);
@@ -355,6 +364,7 @@ struct GemmTestbed {
         algorithm(algorithm_),
         batch_count(1),
         partitionK_count(1),
+        partitionK_multiple(1),
         batch_stride_A(static_cast<long long int>(0)),
         batch_stride_B(static_cast<long long int>(0)),
         batch_stride_C(static_cast<long long int>(0)) {
@@ -389,13 +399,19 @@ struct GemmTestbed {
         algorithm(algorithm_),
         batch_count(1),
         partitionK_count(1),
+        partitionK_multiple(1),
         batch_stride_A(static_cast<long long int>(0)),
         batch_stride_B(static_cast<long long int>(0)),
         batch_stride_C(static_cast<long long int>(0)) {
+    
+    #if CUTLASS_ENABLE_CUBLAS
     status = cublasCreate(&handle);
     if (status != CUBLAS_STATUS_SUCCESS) {
       throw cutlass::cuda_exception("Failed to create CUBLAS handle");
     }
+    #else
+    status = CUBLAS_STATUS_NOT_INITIALIZED;
+    #endif
 
     resize(A, M_, K_, layout_a, lda);
     resize(B, K_, N_, layout_b, ldb);
@@ -428,6 +444,7 @@ struct GemmTestbed {
         algorithm(algorithm_),
         batch_count(1),
         partitionK_count(1),
+        partitionK_multiple(1),
         batch_stride_A(static_cast<long long int>(0)),
         batch_stride_B(static_cast<long long int>(0)),
         batch_stride_C(static_cast<long long int>(0)) {
@@ -462,12 +479,17 @@ struct GemmTestbed {
         beta(beta_),
         algorithm(algorithm_),
         batch_count(batch_count_),
-        partitionK_count(1) {
+        partitionK_count(1),
+        partitionK_multiple(1) {
 
+    #if CUTLASS_ENABLE_CUBLAS
     status = cublasCreate(&handle);
     if (status != CUBLAS_STATUS_SUCCESS) {
       throw cutlass::cuda_exception("Failed to create CUBLAS handle");
     }
+    #else
+    status = CUBLAS_STATUS_NOT_INITIALIZED;
+    #endif
 
     resize(A, M_, K_ * batch_count, layout_a);
     resize(B, K_ * batch_count, N_, layout_b);
@@ -491,6 +513,7 @@ struct GemmTestbed {
   GemmTestbed(int M_,
               int N_,
               std::pair<int, int> K_pair_, /*(k, partitionK_count)*/
+              int partitionK_multiple_, /*each partition should be mulitiple of partitionK_multiple*/
               cublasOperation_t layout_a,
               cublasOperation_t layout_b,
               Scalar alpha_ = Scalar(1),
@@ -504,12 +527,18 @@ struct GemmTestbed {
     beta(beta_),
     algorithm(algorithm_),
     batch_count(1),
-    partitionK_count(K_pair_.second) {
+    partitionK_count(K_pair_.second),
+    partitionK_multiple(partitionK_multiple_) {
 
+    #if CUTLASS_ENABLE_CUBLAS
     status = cublasCreate(&handle);
     if (status != CUBLAS_STATUS_SUCCESS) {
       throw cutlass::cuda_exception("Failed to create CUBLAS handle");
     }
+    #else
+    status = CUBLAS_STATUS_NOT_INITIALIZED;
+    #endif
+
     resize(A, M_, K_pair_.first, layout_a);
     resize(B, K_pair_.first, N_, layout_b);
     resize(C_initial, M_, N_ * partitionK_count, layout_c);
@@ -521,6 +550,7 @@ struct GemmTestbed {
     // we can use a combination of batched stried gemm and regular gemm
     // to simulation partitionedK, which is what we will do for reference code
     int partitionK_size = K() / partitionK_count;
+    partitionK_size = partitionK_size - (partitionK_size % partitionK_multiple);
     batch_stride_A = (layout_a == CUBLAS_OP_N) ? M_ * partitionK_size : partitionK_size;
     batch_stride_B = (layout_b == CUBLAS_OP_N) ? partitionK_size : partitionK_size * N_;
     batch_stride_C = M_ * N_;
@@ -528,9 +558,11 @@ struct GemmTestbed {
 
   /// Destructs the GEMM testbed
   ~GemmTestbed() {
+    #if CUTLASS_ENABLE_CUBLAS
     if (status != CUBLAS_STATUS_NOT_INITIALIZED) {
       status = cublasDestroy(handle);
     }
+    #endif
   }
 
   /// Returns true if the last CUBLAS call returned successfully
@@ -623,15 +655,16 @@ struct GemmTestbed {
     // Initialize the source matrix with a uniform distribution
     cutlass::Distribution dist;
     dist.set_uniform(-8, 8);
-    
+
     cutlass::reference::host::TensorInitialize(A.host_view(), seed, dist);
     cutlass::reference::host::TensorInitialize(B.host_view(), seed + 11, dist);
     cutlass::reference::host::TensorInitialize(C_initial.host_view(), seed + 13, dist);
-    
+
     A.sync_device();
     B.sync_device();
     C_initial.sync_device();
 
+    computed.fill(0);
   }
 
   /// Initializes binary data
@@ -673,6 +706,7 @@ struct GemmTestbed {
 
   /// Excutes an equivalent GEMM using cuBLAS
   bool execute_cublas() {
+    #if CUTLASS_ENABLE_CUBLAS
     if (partitionK_count == 1) {
         if (batch_count == 1) {
           status = cublasGemmEx(handle,
@@ -727,6 +761,7 @@ struct GemmTestbed {
       //first call strided batched gemm
 
       int partitionK_size = K() / partitionK_count;
+      partitionK_size = partitionK_size - (partitionK_size % partitionK_multiple);
       //int lastK_size = (K() % partitionK_size) + partitionK_size;
       int lastK_size = K() - partitionK_size * (partitionK_count - 1);
       status = cublasGemmStridedBatchedTemplate(handle,
@@ -770,6 +805,9 @@ struct GemmTestbed {
       return status == CUBLAS_STATUS_SUCCESS;
 
     }
+    #else
+    return false;
+    #endif
   }
 
   /// Helper function to use cublasGemmStridedBatched
@@ -892,49 +930,65 @@ struct GemmTestbed {
 
   /// Verifies the contents of computed equal cuBLAS
   bool verify_with_cublas(bool save_on_error = true, bool always_print = false) {
+
+    bool passed = false;
+
+    #if CUTLASS_ENABLE_CUBLAS
     compute_cublas();
 
     ref_cublas.sync_host();
     computed.sync_host();
 
-
-    bool passed = computed.bit_equals(ref_cublas);
+    passed = computed.bit_equals(ref_cublas);
 
     if ((!passed && save_on_error) || always_print) {
       save_workspace(computed, ref_cublas);
     }
+
+    #endif
     return passed;
   }
 
   /// Verifies the host computation with cuBLAS
   bool verify_host_with_cublas(bool save_on_error = true, bool always_print = false) {
+
+    bool passed = false;
+
+    #if CUTLASS_ENABLE_CUBLAS
+
     compute_host();
     compute_cublas();
     ref_cublas.sync_host();
 
-    bool passed = ref_host.bit_equals(ref_cublas);
+    passed = ref_host.bit_equals(ref_cublas);
 
     if ((!passed && save_on_error) || always_print) {
       save_workspace(ref_host, ref_cublas);
     }
 
+    #endif
+
     return passed;
   }
 
   /// Verifies the reference implementation with cuBLAS
   bool verify_reference_with_cublas(bool save_on_error = true, bool always_print = false) {
 
+    bool passed = false;
+
+    #if CUTLASS_ENABLE_CUBLAS
     compute_device_reference();
     ref_device.sync_host();
 
     compute_cublas();
     ref_cublas.sync_host();
 
-    bool passed = ref_device.bit_equals(ref_cublas);
+    passed = ref_device.bit_equals(ref_cublas);
 
     if ((!passed && save_on_error) || always_print) {
       save_workspace(ref_device, ref_cublas);
     }
+    #endif
 
     return passed;
   }
@@ -948,15 +1002,26 @@ struct GemmTestbed {
     // verify on host
     passed = (passed && verify_with_host());
 
+    #if CUTLASS_ENABLE_CUBLAS
     // verify with cublas
     passed = (passed && verify_with_cublas());
+    #endif
 
     return passed;
   }
 
-  bool has_cublas_support() const { return cutlass::platform::is_same<Accumulator, Scalar>::value; }
+  bool has_cublas_support() const {
+    #if CUTLASS_ENABLE_CUBLAS
+    return cutlass::platform::is_same<Accumulator, Scalar>::value;
+    #else
+    return false;
+    #endif
+  }
 };
 
+//////////////////////////////////////////////////////////////////////////////////////////
+
+//////////////////////////////////////////////////////////////////////////////////////////
 //
 //specialization for cublasGemmStridedBatchedTemplate
 template<> inline cublasStatus_t GemmTestbed<float, float, float, float, float>::cublasGemmStridedBatchedTemplate(cublasHandle_t handle,
@@ -977,6 +1042,7 @@ template<> inline cublasStatus_t GemmTestbed<float, float, float, float, float>:
                                                                                                     int ldc,
                                                                                                     long long int stride_C,
                                                                                                     int batchCount) {
+  #if CUTLASS_ENABLE_CUBLAS
   return cublasSgemmStridedBatched(handle,
     transa,
     transb,
@@ -993,6 +1059,9 @@ template<> inline cublasStatus_t GemmTestbed<float, float, float, float, float>:
     ldc,
     stride_C,
     batchCount);
+  #else
+  return CUBLAS_STATUS_NOT_SUPPORTED;
+  #endif
 }
 
 template<> inline cublasStatus_t GemmTestbed<double, double, double, double, double>::cublasGemmStridedBatchedTemplate(cublasHandle_t handle,
@@ -1013,6 +1082,7 @@ template<> inline cublasStatus_t GemmTestbed<double, double, double, double, dou
                                                                                                                 int ldc,
                                                                                                                 long long int stride_C,
                                                                                                                 int batchCount) {
+  #if CUTLASS_ENABLE_CUBLAS
   return cublasDgemmStridedBatched(handle,
     transa,
     transb,
@@ -1029,6 +1099,9 @@ template<> inline cublasStatus_t GemmTestbed<double, double, double, double, dou
     ldc,
     stride_C,
     batchCount);
+  #else
+  return CUBLAS_STATUS_NOT_SUPPORTED;
+  #endif
 }
 
 template<> inline cublasStatus_t GemmTestbed<cutlass::half_t, cutlass::half_t, cutlass::half_t, cutlass::half_t, cutlass::half_t>::cublasGemmStridedBatchedTemplate(cublasHandle_t handle,
@@ -1049,6 +1122,7 @@ template<> inline cublasStatus_t GemmTestbed<cutlass::half_t, cutlass::half_t, c
                                                                                                       int ldc,
                                                                                                       long long int stride_C,
                                                                                                       int batchCount) {
+  #if CUTLASS_ENABLE_CUBLAS
   half temp_alpha = alpha->operator half();
   half temp_beta = beta->operator half();
   return cublasHgemmStridedBatched(handle,
@@ -1067,6 +1141,9 @@ template<> inline cublasStatus_t GemmTestbed<cutlass::half_t, cutlass::half_t, c
     ldc,
     stride_C,
     batchCount);
+  #else
+  return CUBLAS_STATUS_NOT_SUPPORTED;
+  #endif
 }
 
 template<> inline cublasStatus_t GemmTestbed<cutlass::half_t, cutlass::half_t, cutlass::half_t, float, float>::cublasGemmStridedBatchedTemplate(cublasHandle_t handle,
@@ -1087,6 +1164,7 @@ template<> inline cublasStatus_t GemmTestbed<cutlass::half_t, cutlass::half_t, c
                                                                                                       int ldc,
                                                                                                       long long int stride_C,
                                                                                                       int batchCount) {
+  #if CUTLASS_ENABLE_CUBLAS
   return cublasGemmStridedBatchedEx(handle,
     transa,
     transb,
@@ -1108,5 +1186,8 @@ template<> inline cublasStatus_t GemmTestbed<cutlass::half_t, cutlass::half_t, c
     batchCount,
     cutlass::TypeTraits<float>::cublas_type,
     CUBLAS_GEMM_DEFAULT);
+  #else
+  return CUBLAS_STATUS_NOT_SUPPORTED;
+  #endif
 }
 }  // namespace test

tools/test/unit/gemm/hgemm_128x128x16.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/hgemm_128x128x8.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/hgemm_128x32x8.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/hgemm_128x64x8.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/igemm_128x128x32.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -22,6 +22,9 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  **************************************************************************************************/
+
+#if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))
+
 #include "cutlass_unit_test.h"
 #include "cutlass/gemm/gemm.h"
 #include "cutlass/gemm/igemm_traits.h"
@@ -355,3 +358,5 @@ TEST(Igemm_128x128x32, igemm_256x256x64_tt) {
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#endif // if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))

tools/test/unit/gemm/igemm_128x128x32_float.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -22,6 +22,9 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  **************************************************************************************************/
+
+#if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))
+
 #include "cutlass_unit_test.h"
 #include "cutlass/gemm/gemm.h"
 #include "cutlass/gemm/igemm_traits.h"
@@ -356,3 +359,5 @@ TEST(Igemm_128x128x32_float, igemm_256x256x64_tt) {
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#endif // if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))

tools/test/unit/gemm/igemm_128x128x32_int8.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -22,6 +22,9 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  **************************************************************************************************/
+
+#if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))
+
 #include "cutlass_unit_test.h"
 #include "cutlass/gemm/gemm.h"
 #include "cutlass/gemm/igemm_traits.h"
@@ -357,3 +360,5 @@ TEST(Igemm_128x128x32_int8, igemm_256x256x64_tt) {
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#endif // if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))

tools/test/unit/gemm/igemm_128x32x32.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -22,6 +22,9 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  **************************************************************************************************/
+
+#if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))
+
 #include "cutlass_unit_test.h"
 #include "cutlass/gemm/gemm.h"
 #include "cutlass/gemm/igemm_traits.h"
@@ -358,3 +361,5 @@ TEST(Igemm_128x32x32, igemm_256x128x32_tt) {
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#endif // if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))

tools/test/unit/gemm/igemm_128x64x32.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -22,6 +22,9 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  **************************************************************************************************/
+
+#if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))
+
 #include "cutlass_unit_test.h"
 #include "cutlass/gemm/gemm.h"
 #include "cutlass/gemm/igemm_traits.h"
@@ -358,3 +361,5 @@ TEST(Igemm_128x64x32, igemm_256x128x64_tt) {
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#endif // if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))

tools/test/unit/gemm/igemm_32x32x128.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -22,6 +22,9 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  **************************************************************************************************/
+
+#if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))
+
 #include "cutlass_unit_test.h"
 #include "cutlass/gemm/gemm.h"
 #include "cutlass/gemm/igemm_traits.h"
@@ -236,3 +239,5 @@ TEST(Igemm_32x32x128, igemm_32x32x128_tt) {
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#endif // if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))

tools/test/unit/gemm/partitionedK_sgemm_128x128x8.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
-* Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+* Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification, are permitted
 * provided that the following conditions are met:

tools/test/unit/gemm/partitionedK_volta884_hgemm.cu

@@ -0,0 +1,293 @@
+/***************************************************************************************************
+* Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without modification, are permitted
+* provided that the following conditions are met:
+*     * Redistributions of source code must retain the above copyright notice, this list of
+*       conditions and the following disclaimer.
+*     * Redistributions in binary form must reproduce the above copyright notice, this list of
+*       conditions and the following disclaimer in the documentation and/or other materials
+*       provided with the distribution.
+*     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+*       to endorse or promote products derived from this software without specific prior written
+*       permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*
+**************************************************************************************************/
+#include <cublas_v2.h>
+#include <cstring>
+#include "cutlass_unit_test.h"
+
+#include "tools/util/half.h"
+#include "tools/util/host_tensor.h"
+#include "tools/util/tensor_view_io.h"
+
+#include "cutlass/gemm/volta884_gemm_traits.h"
+#include "cutlass/gemm/gemm.h"
+
+#include "tools/test/unit/gemm/gemm_testbed.h"
+#include "tools/test/unit/gemm/run_gemm.h"
+
+#if CUTLASS_ENABLE_TENSOR_CORE_MMA
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_h884gemm_partitionedK_64x64x32, volta884_h884gemm_128x256x88x10_nn) {
+  /*
+  for example
+  partitionedK gemm, m = 128, n = 256, overall_K = 88, partitionK_count = 10
+  for the first 9 partition k = overall_k / partitionK_count = 8
+  for the last partition last_k = overall_k - (partitionK_count - 1) * k = 16
+  for volta884 it is safe to make sure leading dim are multiple of 8
+  */
+
+  int m = 128;
+  int n = 256;
+  int overall_k = 88;
+  int partitionK_count = 10;
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+
+  run_partitioned_k_gemm<GemmTraits>(m, n, overall_k, partitionK_count);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_h884gemm_partitionedK_64x64x32, volta884_h884gemm_128x256x88x10_nt) {
+  /*
+  for example
+  partitionedK gemm, m = 128, n = 256, overall_K = 88, partitionK_count = 10
+  for the first 9 partition k = overall_k / partitionK_count = 8
+  for the last partition last_k = overall_k - (partitionK_count - 1) * k = 16
+  for volta884 it is safe to make sure leading dim are multiple of 8
+  */
+
+  int m = 128;
+  int n = 256;
+  int overall_k = 88;
+  int partitionK_count = 10;
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+
+  run_partitioned_k_gemm<GemmTraits>(m, n, overall_k, partitionK_count);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_h884gemm_partitionedK_64x64x32, volta884_h884gemm_128x256x88x10_tn) {
+  /*
+  for example
+  partitionedK gemm, m = 128, n = 256, overall_K = 88, partitionK_count = 10
+  for the first 9 partition k = overall_k / partitionK_count = 8
+  for the last partition last_k = overall_k - (partitionK_count - 1) * k = 16
+  for volta884 it is safe to make sure leading dim are multiple of 8
+  */
+
+  int m = 128;
+  int n = 256;
+  int overall_k = 88;
+  int partitionK_count = 10;
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+
+  run_partitioned_k_gemm<GemmTraits>(m, n, overall_k, partitionK_count);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_h884gemm_partitionedK_64x64x32, volta884_h884gemm_128x256x88x10_tt) {
+  /*
+  for example
+  partitionedK gemm, m = 128, n = 256, overall_K = 88, partitionK_count = 10
+  for the first 9 partition k = overall_k / partitionK_count = 8
+  for the last partition last_k = overall_k - (partitionK_count - 1) * k = 16
+  for volta884 it is safe to make sure leading dim are multiple of 8
+  */
+
+  int m = 128;
+  int n = 256;
+  int overall_k = 88;
+  int partitionK_count = 10;
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+
+  run_partitioned_k_gemm<GemmTraits>(m, n, overall_k, partitionK_count);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_h884gemm_partitionedK_64x64x32, volta884_h884gemm_128x256x128x10_nn) {
+  /*
+  for example
+  partitionedK gemm, m = 128, n = 256, overall_K = 128, partitionK_count = 10
+  for the first 9 partition k = overall_k / partitionK_count = 12. 
+  But if we require the partition mulitple to be 8, the first 9 partition 
+  k = k - (k % partition_mulitiple) = 8
+  for the last partition last_k = overall_k - (partitionK_count - 1) * k = 56
+  for volta884 it is safe to make sure leading dim are multiple of 8
+  */
+
+  int m = 128;
+  int n = 256;
+  int overall_k = 128;
+  int partitionK_count = 10;
+  int partitionK_multiple = 8;
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+
+  run_partitioned_k_gemm<GemmTraits>(m, n, overall_k, partitionK_count, partitionK_multiple);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_h884gemm_partitionedK_64x64x32, volta884_h884gemm_128x256x128x10_nt) {
+  /*
+  for example
+  partitionedK gemm, m = 128, n = 256, overall_K = 128, partitionK_count = 10
+  for the first 9 partition k = overall_k / partitionK_count = 12.
+  But if we require the partition mulitple to be 8, the first 9 partition
+  k = k - (k % partition_mulitiple) = 8
+  for the last partition last_k = overall_k - (partitionK_count - 1) * k = 56
+  for volta884 it is safe to make sure leading dim are multiple of 8
+  */
+
+  int m = 128;
+  int n = 256;
+  int overall_k = 128;
+  int partitionK_count = 10;
+  int partitionK_multiple = 8;
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+
+  run_partitioned_k_gemm<GemmTraits>(m, n, overall_k, partitionK_count, partitionK_multiple);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_h884gemm_partitionedK_64x64x32, volta884_h884gemm_128x256x128x10_tn) {
+  /*
+  for example
+  partitionedK gemm, m = 128, n = 256, overall_K = 128, partitionK_count = 10
+  for the first 9 partition k = overall_k / partitionK_count = 12.
+  But if we require the partition mulitple to be 8, the first 9 partition
+  k = k - (k % partition_mulitiple) = 8
+  for the last partition last_k = overall_k - (partitionK_count - 1) * k = 56
+  for volta884 it is safe to make sure leading dim are multiple of 8
+  */
+
+  int m = 128;
+  int n = 256;
+  int overall_k = 128;
+  int partitionK_count = 10;
+  int partitionK_multiple = 8;
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+
+  run_partitioned_k_gemm<GemmTraits>(m, n, overall_k, partitionK_count, partitionK_multiple);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_h884gemm_partitionedK_64x64x32, volta884_h884gemm_128x256x128x10_tt) {
+  /*
+  for example
+  partitionedK gemm, m = 128, n = 256, overall_K = 128, partitionK_count = 10
+  for the first 9 partition k = overall_k / partitionK_count = 12.
+  But if we require the partition mulitple to be 8, the first 9 partition
+  k = k - (k % partition_mulitiple) = 8
+  for the last partition last_k = overall_k - (partitionK_count - 1) * k = 56
+  for volta884 it is safe to make sure leading dim are multiple of 8
+  */
+
+  int m = 128;
+  int n = 256;
+  int overall_k = 128;
+  int partitionK_count = 10;
+  int partitionK_multiple = 8;
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+
+  run_partitioned_k_gemm<GemmTraits>(m, n, overall_k, partitionK_count, partitionK_multiple);
+}
+
+#endif // if defined(CUTLASS_ENABLE_TENSOR_CORE_MMA)

tools/test/unit/gemm/run_gemm.h

@@ -44,7 +44,9 @@ static void run_gemm(
     typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type beta =
         typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(0.0f)) {
 
-  typedef typename GemmTraits_::KernelClass Gemm;
+  //typedef typename GemmTraits_::KernelClass Gemm;
+  typedef cutlass::gemm::Gemm<GemmTraits_> Gemm;
+
   typename Gemm::Params params;
 
   test::GemmTestbed<
@@ -106,6 +108,8 @@ static void run_gemm(
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
 template <typename GemmTraits_>
 static void run_gemm(
     int m,
@@ -115,8 +119,8 @@ static void run_gemm(
         typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(1.0f),
     typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type beta =
         typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(0.0f)) {
-  //typedef cutlass::gemm::Gemm<GemmTraits_> Gemm;
-  typedef typename GemmTraits_::KernelClass Gemm;
+  typedef cutlass::gemm::Gemm<GemmTraits_> Gemm;
+  //typedef typename GemmTraits_::KernelClass Gemm;
   typename Gemm::Params params;
 
   typedef test::GemmTestbed<
@@ -185,8 +189,8 @@ static void run_batched_strided_gemm(
         typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(1.0f),
     typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type beta =
         typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(0.0f)) {
-  //typedef cutlass::gemm::Gemm<GemmTraits_> Gemm;
-  typedef typename GemmTraits_::KernelClass Gemm;
+  typedef cutlass::gemm::Gemm<GemmTraits_> Gemm;
+  //typedef typename GemmTraits_::KernelClass Gemm;
   typename Gemm::Params params;
   test::GemmTestbed<
       typename test::GemmTestbedTraits<
@@ -254,6 +258,7 @@ template <typename GemmTraits_, typename ReductionTraits_>
 static void run_splitK_gemm(int m,
   int n,
   int k,
+  int partitionK_multiple = 1, /*requires each partition to be mulitple of partitionK_multiple*/
   typename test::GemmTestbedTraits<typename ReductionTraits_::ScalarAlphaBeta>::host_type alpha =
       typename test::GemmTestbedTraits<typename ReductionTraits_::ScalarAlphaBeta>::host_type(1.0f),
   typename test::GemmTestbedTraits<typename ReductionTraits_::ScalarAlphaBeta>::host_type beta =
@@ -283,11 +288,12 @@ static void run_splitK_gemm(int m,
 
   // create a device gemm
   typedef cutlass::gemm::SplitkPIGemmTraits<GemmTraits_, ReductionTraits_> deviceGemmTraits;
-  typedef typename deviceGemmTraits::KernelClass deviceGemm;
+  //typedef typename deviceGemmTraits::KernelClass deviceGemm;
+  typedef typename cutlass::gemm::DeviceGemm<deviceGemmTraits> deviceGemm;
   typename deviceGemm::Params deviceGemmParams(testbed.M(), testbed.N(), testbed.K());
 
   // query if workspace is needed
-  int workspace_size = deviceGemmParams.required_workspace_memory_in_byte();
+  size_t workspace_size = deviceGemmParams.required_workspace_memory_in_byte();
   typename test::GemmTestbedTraits<typename GemmTraits_::GemmConfig::ScalarD>::device_type
     *workspace_ptr = 0;
   if (workspace_size != 0) {
@@ -306,7 +312,8 @@ static void run_splitK_gemm(int m,
     testbed.ldc(),
     testbed.ptr_computed(),
     testbed.ldc(),
-    workspace_ptr);
+    workspace_ptr,
+    partitionK_multiple);
 
 
   deviceGemm::launch(deviceGemmParams);
@@ -337,12 +344,13 @@ static void run_partitioned_k_gemm(
   int n,
   int k,
   int partitionK_count,
+  int partitionK_multiple = 1, //requires each partition to be multiples of partitionK_multiple
   typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type alpha =
   typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(1.0f),
   typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type beta =
   typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(0.0f)) {
-  //typedef cutlass::gemm::Gemm<GemmTraits_> Gemm;
-  typedef typename GemmTraits_::KernelClass Gemm;
+  typedef cutlass::gemm::Gemm<GemmTraits_> Gemm;
+  //typedef typename GemmTraits_::KernelClass Gemm;
   typename Gemm::Params params;
   test::GemmTestbed<
     typename test::GemmTestbedTraits<
@@ -358,6 +366,7 @@ static void run_partitioned_k_gemm(
     testbed(m,
       n,
       std::make_pair(k, partitionK_count),
+      partitionK_multiple,
       test::convert(GemmTraits_::kLayoutA),
       test::convert(GemmTraits_::kLayoutB),
       alpha,
@@ -383,7 +392,8 @@ static void run_partitioned_k_gemm(
     testbed.ldc(),
     testbed.ptr_computed(),
     testbed.ldc(),
-    partitionK_count);
+    partitionK_count,
+    partitionK_multiple);
 
   Gemm::launch(params);
 

tools/test/unit/gemm/sgemm_128x128x16.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/sgemm_128x128x8.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/sgemm_128x32x16.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/sgemm_128x32x8.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/sgemm_128x64x16.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/sgemm_128x64x8.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/sgemm_32x128x8.cu

@@ -0,0 +1,236 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+#include "cutlass_unit_test.h"
+#include "cutlass/gemm/gemm.h"
+#include "cutlass/gemm/sgemm_traits.h"
+#include "tools/test/unit/gemm/gemm_testbed.h"
+#include "tools/test/unit/gemm/run_gemm.h"
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_32x128x1_nt) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
+                                     cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 32>,
+                                     cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+      SgemmTraits;
+  run_gemm<SgemmTraits>(32, 128, 1);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_32x128x8_nt) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(32, 128, 8);
+}
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_32x128x16_nt) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(32, 128, 16);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_32x256x16_nt) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(32, 256, 16);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_64x256x16_nt) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(64, 256, 16);
+}
+
+//NN
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_32x128x1_nn) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(32, 128, 1);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_32x128x8_nn) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(32, 128, 8);
+}
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_32x128x16_nn) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(32, 128, 16);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_32x256x16_nn) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(32, 256, 16);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_64x256x16_nn) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(64, 256, 16);
+}
+
+//TN
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_32x128x1_tn) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(32, 128, 1);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_32x128x8_tn) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(32, 128, 8);
+}
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_32x128x16_tn) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(32, 128, 16);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_32x256x16_tn) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(32, 256, 16);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_64x256x16_tn) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(64, 256, 16);
+}
+
+//TT
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_32x128x1_tt) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(32, 128, 1);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_32x128x8_tt) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(32, 128, 8);
+}
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_32x128x16_tt) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(32, 128, 16);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_32x256x16_tt) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(32, 256, 16);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Sgemm_32x128x8, sgemm_64x256x16_tt) {
+  typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 32>,
+    cutlass::gemm::LinearScaling<float>, cutlass::Shape<8, 8, 4> >
+    SgemmTraits;
+  run_gemm<SgemmTraits>(64, 256, 16);
+}

tools/test/unit/gemm/sgemm_64x128x16.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/sgemm_64x128x8.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/sgemm_64x32x16.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/sgemm_64x32x8.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/sgemm_64x64x16.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/sgemm_64x64x8.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/sgemm_threadblock_swizzle_nn.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/sgemm_threadblock_swizzle_nt.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/sgemm_threadblock_swizzle_tn.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/sgemm_threadblock_swizzle_tt.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/unit/gemm/splitK_dgemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
-* Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+* Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification, are permitted
 * provided that the following conditions are met:
@@ -54,7 +54,7 @@ TEST(SplitK_dgemm_128x128x8_splits16, dgemm_128x256x512_nn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -81,7 +81,7 @@ TEST(SplitK_dgemm_128x128x8_splits16, dgemm_128x256x512_nt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -108,7 +108,7 @@ TEST(SplitK_dgemm_128x128x8_splits16, dgemm_128x256x512_tn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -135,7 +135,7 @@ TEST(SplitK_dgemm_128x128x8_splits16, dgemm_128x256x512_tt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -162,7 +162,7 @@ TEST(SplitK_dgemm_128x128x8_splits16, dgemm_128x256x500_nn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -189,7 +189,7 @@ TEST(SplitK_dgemm_128x128x8_splits16, dgemm_128x256x500_nt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -216,7 +216,7 @@ TEST(SplitK_dgemm_128x128x8_splits16, dgemm_128x256x500_tn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -243,5 +243,5 @@ TEST(SplitK_dgemm_128x128x8_splits16, dgemm_128x256x500_tt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }

tools/test/unit/gemm/splitK_fp16_sgemm_fp16.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
-* Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+* Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification, are permitted
 * provided that the following conditions are met:
@@ -69,7 +69,7 @@ TEST(SplitK_fp16_sgemm_fp16_alphabetaFloat_128x128x8_splits16, sgemm_128x256x512
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f, true/*use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -103,7 +103,7 @@ TEST(SplitK_fp16_sgemm_fp16_alphabetaFloat_128x128x8_splits16, sgemm_128x256x512
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f, true/*use host reference*/);
 }
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
@@ -136,7 +136,7 @@ TEST(SplitK_fp16_sgemm_fp16_alphabetaFloat_128x128x8_splits16, sgemm_128x256x512
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f, true/*use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -170,7 +170,7 @@ TEST(SplitK_fp16_sgemm_fp16_alphabetaFloat_128x128x8_splits16, sgemm_128x256x512
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f, true/*use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -204,7 +204,7 @@ TEST(SplitK_fp16_sgemm_fp16_alphabetaFloat_128x128x8_splits16, sgemm_128x256x500
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f, true/*use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -238,7 +238,7 @@ TEST(SplitK_fp16_sgemm_fp16_alphabetaFloat_128x128x8_splits16, sgemm_128x256x500
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f, true/*use host reference*/);
 }
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
@@ -271,7 +271,7 @@ TEST(SplitK_fp16_sgemm_fp16_alphabetaFloat_128x128x8_splits16, sgemm_128x256x500
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f, true/*use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -305,7 +305,7 @@ TEST(SplitK_fp16_sgemm_fp16_alphabetaFloat_128x128x8_splits16, sgemm_128x256x500
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f, true/*use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -339,7 +339,7 @@ TEST(SplitK_fp16_sgemm_fp16_alphabetaFp16_128x128x8_splits16, sgemm_128x256x512_
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f, true/*use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -373,7 +373,7 @@ TEST(SplitK_fp16_sgemm_fp16_alphabetaFp16_128x128x8_splits16, sgemm_128x256x512_
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f, true/*use host reference*/);
 }
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
@@ -406,7 +406,7 @@ TEST(SplitK_fp16_sgemm_fp16_alphabetaFp16_128x128x8_splits16, sgemm_128x256x512_
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f, true/*use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -440,7 +440,7 @@ TEST(SplitK_fp16_sgemm_fp16_alphabetaFp16_128x128x8_splits16, sgemm_128x256x512_
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f, true/*use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -474,7 +474,7 @@ TEST(SplitK_fp16_sgemm_fp16_alphabetaFp16_128x128x8_splits16, sgemm_128x256x500_
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f, true/*use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -508,7 +508,7 @@ TEST(SplitK_fp16_sgemm_fp16_alphabetaFp16_128x128x8_splits16, sgemm_128x256x500_
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f, true/*use host reference*/);
 }
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
@@ -541,7 +541,7 @@ TEST(SplitK_fp16_sgemm_fp16_alphabetaFp16_128x128x8_splits16, sgemm_128x256x500_
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f, true/*use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -575,5 +575,5 @@ TEST(SplitK_fp16_sgemm_fp16_alphabetaFp16_128x128x8_splits16, sgemm_128x256x500_
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f, true/*use host reference*/);
 }

tools/test/unit/gemm/splitK_hgemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
-* Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+* Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification, are permitted
 * provided that the following conditions are met:
@@ -54,7 +54,7 @@ TEST(SplitK_hgemm_128x128x8_splits16, hgemm_128x256x64_nn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 1.0f, 0.0f);
+  run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 1.0f, 0.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -81,7 +81,7 @@ TEST(SplitK_hgemm_128x128x8_splits16, hgemm_128x256x64_nt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -108,7 +108,7 @@ TEST(SplitK_hgemm_128x128x8_splits16, hgemm_128x256x64_tn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -135,7 +135,7 @@ TEST(SplitK_hgemm_128x128x8_splits16, hgemm_128x256x64_tt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -162,7 +162,7 @@ TEST(SplitK_hgemm_128x128x8_splits16, hgemm_128x256x66_nn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 1.0f, 0.0f);
+  run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 1.0f, 0.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -189,7 +189,7 @@ TEST(SplitK_hgemm_128x128x8_splits16, hgemm_128x256x66_nt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -216,7 +216,7 @@ TEST(SplitK_hgemm_128x128x8_splits16, hgemm_128x256x66_tn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -243,6 +243,6 @@ TEST(SplitK_hgemm_128x128x8_splits16, hgemm_128x256x66_tt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 

tools/test/unit/gemm/splitK_igemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
-* Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+* Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification, are permitted
 * provided that the following conditions are met:
@@ -22,6 +22,9 @@
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 **************************************************************************************************/
+
+#if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))
+
 #include "cutlass_unit_test.h"
 #include "cutlass/gemm/gemm.h"
 #include "cutlass/gemm/igemm_traits.h"
@@ -83,7 +86,7 @@ TEST(SplitK_igemm_128x128x32_splits16, igemm_128x256x512_nt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 2, 1, true /*use host reference*/);
+  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2, 1, true /*use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -111,7 +114,7 @@ TEST(SplitK_igemm_128x128x32_splits16, igemm_128x256x512_tn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 2, 1, true /*use host reference*/);
+  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2, 1, true /*use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -139,7 +142,7 @@ TEST(SplitK_igemm_128x128x32_splits16, igemm_128x256x512_tt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 2, 1, true /*use host reference*/);
+  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2, 1, true /*use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -167,7 +170,7 @@ TEST(SplitK_igemm_128x128x32_splits16, igemm_1024x64x4096_nn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1, 0, false /*not use host reference*/);
+  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 1, 0, false /*not use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -195,7 +198,7 @@ TEST(SplitK_igemm_128x128x32_splits16, igemm_1024x64x4096_nt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1, 0, false /*not use host reference*/);
+  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 1, 0, false /*not use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -223,7 +226,7 @@ TEST(SplitK_igemm_128x128x32_splits16, igemm_1024x64x4096_tn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1, 0, false /*not use host reference*/);
+  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 1, 0, false /*not use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -251,7 +254,7 @@ TEST(SplitK_igemm_128x128x32_splits16, igemm_1024x64x4096_tt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1, 0, false /*not use host reference*/);
+  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 1, 0, false /*not use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -279,7 +282,7 @@ TEST(SplitK_igemm_128x32x32_splits16, igemm_1024x64x4096_nn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1, 0, false /*not use host reference*/);
+  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 1, 0, false /*not use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -307,7 +310,7 @@ TEST(SplitK_igemm_128x32x32_splits16, igemm_1024x64x4096_nt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1, 0, false /*not use host reference*/);
+  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 1, 0, false /*not use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -335,7 +338,7 @@ TEST(SplitK_igemm_128x32x32_splits16, igemm_1024x64x4096_tn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1, 0, false /*not use host reference*/);
+  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 1, 0, false /*not use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -363,5 +366,7 @@ TEST(SplitK_igemm_128x32x32_splits16, igemm_1024x64x4096_tt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1, 0, false /*not use host reference*/);
+  run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 1, 0, false /*not use host reference*/);
 }
+
+#endif // if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))

tools/test/unit/gemm/splitK_sgemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
-* Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+* Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification, are permitted
 * provided that the following conditions are met:
@@ -54,7 +54,7 @@ TEST(SplitK_sgemm_128x128x8_splits16, sgemm_128x256x512_nn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -81,7 +81,7 @@ TEST(SplitK_sgemm_128x128x8_splits16, sgemm_128x256x512_nt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -108,7 +108,7 @@ TEST(SplitK_sgemm_128x128x8_splits16, sgemm_128x256x512_tn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -135,7 +135,7 @@ TEST(SplitK_sgemm_128x128x8_splits16, sgemm_128x256x512_tt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -162,7 +162,7 @@ TEST(SplitK_sgemm_128x128x8_splits16, sgemm_128x256x500_nn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -189,7 +189,7 @@ TEST(SplitK_sgemm_128x128x8_splits16, sgemm_128x256x500_nt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -216,7 +216,7 @@ TEST(SplitK_sgemm_128x128x8_splits16, sgemm_128x256x500_tn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -243,7 +243,7 @@ TEST(SplitK_sgemm_128x128x8_splits16, sgemm_128x256x500_tt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -270,7 +270,7 @@ TEST(SplitK_sgemm_128x128x8_splits16, sgemm_1024x64x4096_nn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -297,7 +297,7 @@ TEST(SplitK_sgemm_128x128x8_splits16, sgemm_1024x64x4096_nt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -324,7 +324,7 @@ TEST(SplitK_sgemm_128x128x8_splits16, sgemm_1024x64x4096_tn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -351,5 +351,5 @@ TEST(SplitK_sgemm_128x128x8_splits16, sgemm_1024x64x4096_tt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
+  run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f);
 }

tools/test/unit/gemm/splitK_volta884_hgemm.cu

@@ -0,0 +1,507 @@
+/***************************************************************************************************
+* Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without modification, are permitted
+* provided that the following conditions are met:
+*     * Redistributions of source code must retain the above copyright notice, this list of
+*       conditions and the following disclaimer.
+*     * Redistributions in binary form must reproduce the above copyright notice, this list of
+*       conditions and the following disclaimer in the documentation and/or other materials
+*       provided with the distribution.
+*     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+*       to endorse or promote products derived from this software without specific prior written
+*       permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*
+**************************************************************************************************/
+#include <cublas_v2.h>
+#include <cstring>
+#include "cutlass_unit_test.h"
+
+#include "tools/util/half.h"
+#include "tools/util/host_tensor.h"
+#include "tools/util/tensor_view_io.h"
+
+#include "cutlass/gemm/volta884_gemm_traits.h"
+#include "cutlass/gemm/gemm.h"
+
+#include "cutlass/reduction/batched_reduction_traits.h"
+
+#include "tools/test/unit/gemm/gemm_testbed.h"
+#include "tools/test/unit/gemm/run_gemm.h"
+
+#if CUTLASS_ENABLE_TENSOR_CORE_MMA
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_splitK_h884gemm_64x64x32_splits16, volta884_h884gemm_128x256x512_nn) {
+  const int splits_count = 16;
+  const int m = 128;
+  const int n = 256;
+  const int k = 512;
+
+  /*gemm traits*/
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+  /*batched reduction traits*/
+  typedef cutlass::reduction::BatchedReductionTraits<half,
+    half,
+    half,
+    half,
+    half, /*accumulation type*/
+    splits_count,
+    cutlass::Shape<1, 1, 128>,
+    cutlass::Shape<1, 1, 64>,
+    cutlass::Shape<1, 1, 2> >
+    BatchedReductionTraits;
+
+  run_splitK_gemm<GemmTraits, BatchedReductionTraits>(m, n, k, 8/*partitionK_multiple*/, 1.0f, 0.0f);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_splitK_h884gemm_64x64x32_splits16, volta884_h884gemm_128x256x512_nt) {
+  const int splits_count = 16;
+  const int m = 128;
+  const int n = 256;
+  const int k = 512;
+
+  /*gemm traits*/
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+  /*batched reduction traits*/
+  typedef cutlass::reduction::BatchedReductionTraits<half,
+    half,
+    half,
+    half,
+    half, /*accumulation type*/
+    splits_count,
+    cutlass::Shape<1, 1, 128>,
+    cutlass::Shape<1, 1, 64>,
+    cutlass::Shape<1, 1, 2> >
+    BatchedReductionTraits;
+
+  run_splitK_gemm<GemmTraits, BatchedReductionTraits>(m, n, k, 8/*partitionK_multiple*/, 1.0f, 0.0f);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_splitK_h884gemm_64x64x32_splits16, volta884_h884gemm_128x256x512_tn) {
+  const int splits_count = 16;
+  const int m = 128;
+  const int n = 256;
+  const int k = 512;
+
+  /*gemm traits*/
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+  /*batched reduction traits*/
+  typedef cutlass::reduction::BatchedReductionTraits<half,
+    half,
+    half,
+    half,
+    half, /*accumulation type*/
+    splits_count,
+    cutlass::Shape<1, 1, 128>,
+    cutlass::Shape<1, 1, 64>,
+    cutlass::Shape<1, 1, 2> >
+    BatchedReductionTraits;
+
+  run_splitK_gemm<GemmTraits, BatchedReductionTraits>(m, n, k, 8/*partitionK_multiple*/, 1.0f, 0.0f);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_splitK_h884gemm_64x64x32_splits16, volta884_h884gemm_128x256x512_tt) {
+  const int splits_count = 16;
+  const int m = 128;
+  const int n = 256;
+  const int k = 512;
+
+  /*gemm traits*/
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+  /*batched reduction traits*/
+  typedef cutlass::reduction::BatchedReductionTraits<half,
+    half,
+    half,
+    half,
+    half, /*accumulation type*/
+    splits_count,
+    cutlass::Shape<1, 1, 128>,
+    cutlass::Shape<1, 1, 64>,
+    cutlass::Shape<1, 1, 2> >
+    BatchedReductionTraits;
+
+  run_splitK_gemm<GemmTraits, BatchedReductionTraits>(m, n, k, 8/*partitionK_multiple*/, 1.0f, 0.0f);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_splitK_h884gemm_64x64x32_splits10, volta884_h884gemm_128x256x88_nn) {
+  /*
+  m = 128, n = 256, overall_K = 88, splits_count = 10
+  for the first 9 partition k = overall_k / partitionK_count = 8
+  for the last partition last_k = overall_k - (partitionK_count - 1) * k = 16
+  for volta884 it is safe to make sure leading dim are multiple of 8
+  */
+
+  const int splits_count = 10;
+  const int m = 128;
+  const int n = 256;
+  const int k = 88;
+
+  /*gemm traits*/
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+  /*batched reduction traits*/
+  typedef cutlass::reduction::BatchedReductionTraits<half,
+    half,
+    half,
+    half,
+    half, /*accumulation type*/
+    splits_count,
+    cutlass::Shape<1, 1, 128>,
+    cutlass::Shape<1, 1, 64>,
+    cutlass::Shape<1, 1, 2> >
+    BatchedReductionTraits;
+
+  run_splitK_gemm<GemmTraits, BatchedReductionTraits>(m, n, k, 8/*partitionK_multiple*/, 1.0f, 0.0f);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_splitK_h884gemm_64x64x32_splits10, volta884_h884gemm_128x256x88_nt) {
+  /*
+  m = 128, n = 256, overall_K = 88, splits_count = 10
+  for the first 9 partition k = overall_k / partitionK_count = 8
+  for the last partition last_k = overall_k - (partitionK_count - 1) * k = 16
+  for volta884 it is safe to make sure leading dim are multiple of 8
+  */
+  const int splits_count = 10;
+  const int m = 128;
+  const int n = 256;
+  const int k = 88;
+
+  /*gemm traits*/
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+  /*batched reduction traits*/
+  typedef cutlass::reduction::BatchedReductionTraits<half,
+    half,
+    half,
+    half,
+    half, /*accumulation type*/
+    splits_count,
+    cutlass::Shape<1, 1, 128>,
+    cutlass::Shape<1, 1, 64>,
+    cutlass::Shape<1, 1, 2> >
+    BatchedReductionTraits;
+
+  run_splitK_gemm<GemmTraits, BatchedReductionTraits>(m, n, k, 8/*partitionK_multiple*/, 1.0f, 0.0f);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_splitK_h884gemm_64x64x32_splits10, volta884_h884gemm_128x256x88_tn) {
+  /*
+  m = 128, n = 256, overall_K = 88, splits_count = 10
+  for the first 9 partition k = overall_k / partitionK_count = 8
+  for the last partition last_k = overall_k - (partitionK_count - 1) * k = 16
+  for volta884 it is safe to make sure leading dim are multiple of 8
+  */
+  const int splits_count = 10;
+  const int m = 128;
+  const int n = 256;
+  const int k = 88;
+
+  /*gemm traits*/
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+  /*batched reduction traits*/
+  typedef cutlass::reduction::BatchedReductionTraits<half,
+    half,
+    half,
+    half,
+    half, /*accumulation type*/
+    splits_count,
+    cutlass::Shape<1, 1, 128>,
+    cutlass::Shape<1, 1, 64>,
+    cutlass::Shape<1, 1, 2> >
+    BatchedReductionTraits;
+
+  run_splitK_gemm<GemmTraits, BatchedReductionTraits>(m, n, k, 8/*partitionK_multiple*/, 1.0f, 0.0f);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_splitK_h884gemm_64x64x32_splits10, volta884_h884gemm_128x256x88_tt) {
+  /*
+  m = 128, n = 256, overall_K = 88, splits_count = 10
+  for the first 9 partition k = overall_k / partitionK_count = 8
+  for the last partition last_k = overall_k - (partitionK_count - 1) * k = 16
+  for volta884 it is safe to make sure leading dim are multiple of 8
+  */
+  const int splits_count = 10;
+  const int m = 128;
+  const int n = 256;
+  const int k = 88;
+
+  /*gemm traits*/
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+  /*batched reduction traits*/
+  typedef cutlass::reduction::BatchedReductionTraits<half,
+    half,
+    half,
+    half,
+    half, /*accumulation type*/
+    splits_count,
+    cutlass::Shape<1, 1, 128>,
+    cutlass::Shape<1, 1, 64>,
+    cutlass::Shape<1, 1, 2> >
+    BatchedReductionTraits;
+
+  run_splitK_gemm<GemmTraits, BatchedReductionTraits>(m, n, k, 8/*partitionK_multiple*/, 1.0f, 0.0f);
+}
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_splitK_h884gemm_64x64x32_splits10, volta884_h884gemm_128x256x256_nn) {
+  /*
+  m = 128, n = 256, overall_K = 256, splits_count = 10
+  for the first 9 partition k = overall_k / partitionK_count = 25
+  But if we require the partition mulitple to be 8, the first 9 partition
+  k = k - (k % partition_mulitiple) = 24
+  for the last partition last_k = overall_k - (partitionK_count - 1) * k = 40
+  for volta884 it is safe to make sure leading dim are multiple of 8
+  */
+  const int splits_count = 10;
+  const int m = 128;
+  const int n = 256;
+  const int k = 256;
+
+  /*gemm traits*/
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+  /*batched reduction traits*/
+  typedef cutlass::reduction::BatchedReductionTraits<half,
+    half,
+    half,
+    half,
+    half, /*accumulation type*/
+    splits_count,
+    cutlass::Shape<1, 1, 128>,
+    cutlass::Shape<1, 1, 64>,
+    cutlass::Shape<1, 1, 2> >
+    BatchedReductionTraits;
+
+  run_splitK_gemm<GemmTraits, BatchedReductionTraits>(m, n, k, 8/*partitionK_multiple*/, 1.0f, 0.0f);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_splitK_h884gemm_64x64x32_splits10, volta884_h884gemm_128x256x256_nt) {
+  /*
+  m = 128, n = 256, overall_K = 256, splits_count = 10
+  for the first 9 partition k = overall_k / partitionK_count = 25
+  But if we require the partition mulitple to be 8, the first 9 partition
+  k = k - (k % partition_mulitiple) = 24
+  for the last partition last_k = overall_k - (partitionK_count - 1) * k = 40
+  for volta884 it is safe to make sure leading dim are multiple of 8
+  */
+  const int splits_count = 10;
+  const int m = 128;
+  const int n = 256;
+  const int k = 256;
+
+  /*gemm traits*/
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+  /*batched reduction traits*/
+  typedef cutlass::reduction::BatchedReductionTraits<half,
+    half,
+    half,
+    half,
+    half, /*accumulation type*/
+    splits_count,
+    cutlass::Shape<1, 1, 128>,
+    cutlass::Shape<1, 1, 64>,
+    cutlass::Shape<1, 1, 2> >
+    BatchedReductionTraits;
+
+  run_splitK_gemm<GemmTraits, BatchedReductionTraits>(m, n, k, 8/*partitionK_multiple*/, 1.0f, 0.0f);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_splitK_h884gemm_64x64x32_splits10, volta884_h884gemm_128x256x256_tn) {
+  /*
+  m = 128, n = 256, overall_K = 256, splits_count = 10
+  for the first 9 partition k = overall_k / partitionK_count = 25
+  But if we require the partition mulitple to be 8, the first 9 partition
+  k = k - (k % partition_mulitiple) = 24
+  for the last partition last_k = overall_k - (partitionK_count - 1) * k = 40
+  for volta884 it is safe to make sure leading dim are multiple of 8
+  */
+  const int splits_count = 10;
+  const int m = 128;
+  const int n = 256;
+  const int k = 256;
+
+  /*gemm traits*/
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+  /*batched reduction traits*/
+  typedef cutlass::reduction::BatchedReductionTraits<half,
+    half,
+    half,
+    half,
+    half, /*accumulation type*/
+    splits_count,
+    cutlass::Shape<1, 1, 128>,
+    cutlass::Shape<1, 1, 64>,
+    cutlass::Shape<1, 1, 2> >
+    BatchedReductionTraits;
+
+  run_splitK_gemm<GemmTraits, BatchedReductionTraits>(m, n, k, 8/*partitionK_multiple*/, 1.0f, 0.0f);
+}
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_splitK_h884gemm_64x64x32_splits10, volta884_h884gemm_128x256x256_tt) {
+  /*
+  m = 128, n = 256, overall_K = 256, splits_count = 10
+  for the first 9 partition k = overall_k / partitionK_count = 25
+  But if we require the partition mulitple to be 8, the first 9 partition
+  k = k - (k % partition_mulitiple) = 24
+  for the last partition last_k = overall_k - (partitionK_count - 1) * k = 40
+  for volta884 it is safe to make sure leading dim are multiple of 8
+  */
+  const int splits_count = 10;
+  const int m = 128;
+  const int n = 256;
+  const int k = 256;
+
+  /*gemm traits*/
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 64, 64>,
+    half,
+    half,
+    half,
+    2
+  > GemmTraits;
+  /*batched reduction traits*/
+  typedef cutlass::reduction::BatchedReductionTraits<half,
+    half,
+    half,
+    half,
+    half, /*accumulation type*/
+    splits_count,
+    cutlass::Shape<1, 1, 128>,
+    cutlass::Shape<1, 1, 64>,
+    cutlass::Shape<1, 1, 2> >
+    BatchedReductionTraits;
+
+  run_splitK_gemm<GemmTraits, BatchedReductionTraits>(m, n, k, 8/*partitionK_multiple*/, 1.0f, 0.0f);
+}
+
+#endif

tools/test/unit/gemm/splitK_wmma_gemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
-* Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+* Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification, are permitted
 * provided that the following conditions are met:
@@ -64,7 +64,7 @@ TEST(SplitK_wmma_gemm_16x16x32_splits16, wmma_gemm_128x256x512_nn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<WmmaGemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
+  run_splitK_gemm<WmmaGemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 2.0f, 1.0f, true/*use host reference*/);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -99,7 +99,7 @@ TEST(SplitK_wmma_gemm_16x16x32_splits16, wmma_gemm_128x256x512_nt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<WmmaGemmTraits, BatchedReductionTraits>(m, n, k, 1.0f, 0.0f);
+  run_splitK_gemm<WmmaGemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 1.0f, 0.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -134,7 +134,7 @@ TEST(SplitK_wmma_gemm_16x16x32_splits16, wmma_gemm_128x256x512_tn) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<WmmaGemmTraits, BatchedReductionTraits>(m, n, k, 1.0f, 0.0f);
+  run_splitK_gemm<WmmaGemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 1.0f, 0.0f);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -169,7 +169,7 @@ TEST(SplitK_wmma_gemm_16x16x32_splits16, wmma_gemm_128x256x512_tt) {
     cutlass::Shape<1, 1, 2> >
     BatchedReductionTraits;
 
-  run_splitK_gemm<WmmaGemmTraits, BatchedReductionTraits>(m, n, k, 1.0f, 0.0f);
+  run_splitK_gemm<WmmaGemmTraits, BatchedReductionTraits>(m, n, k, 1/*partitionK_multiple*/, 1.0f, 0.0f);
 }
 
 #endif

tools/test/unit/gemm/volta884_gemm_epilogue.cu

@@ -0,0 +1,453 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+/*! \file
+    \brief Implements the epilogue phase of the GEMM kernel that efficiently updates global memory
+   with
+      the computed matrix product.
+*/
+// clang-format off
+
+#include <cublas_v2.h>
+#include <cstring>
+#include "cutlass_unit_test.h"
+
+#include "tools/util/half.h"
+#include "tools/util/host_matrix.h"
+#include "tools/util/tensor_view_io.h"
+
+#include "cutlass/tile_traits_standard.h"
+#include "cutlass/gemm/linear_scaling.h"
+
+#include "cutlass/gemm/volta884_multiplicand.h"
+#include "cutlass/gemm/volta884_multiply_add.h"
+#include "cutlass/gemm/mma_global_stream.h"
+#include "cutlass/gemm/volta884_gemm_epilogue_traits.h"
+#include "cutlass/gemm/volta884_shared_tile.h"
+#include "cutlass/gemm/mma_shared_stream.h"
+#include "cutlass/gemm/mma_epilogue.h"
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#if CUTLASS_ENABLE_TENSOR_CORE_MMA
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/// Kernel that verifies the Volta884 epilogue against the naive epilogue implementation
+template <typename EpilogueTraits, typename AccumulatorType>
+__global__ void test_volta884_epilogue(
+  typename EpilogueTraits::Params params,
+  AccumulatorType *ptr_Product,
+  int ldm,
+  cutlass::Coord<3> problem_size) {
+
+  // Shared memoryallocation
+  __shared__ typename EpilogueTraits::SharedStorage shared_storage;
+
+  // Construct the epilogue
+  cutlass::gemm::MMAEpilogue<EpilogueTraits> epilogue(params, shared_storage, problem_size);
+
+  // Initialize accumulators
+  typedef typename EpilogueTraits::Accumulators Accumulators;
+
+  typedef typename cutlass::gemm::Volta884NaiveEpilogue<
+    AccumulatorType,
+    typename EpilogueTraits::WarpDelta,
+    cutlass::Shape<2,2,2,2> > NaiveEpilogue;
+
+  Accumulators accumulators;
+
+  // Artificially load accumulators with some random matrix product
+  NaiveEpilogue naive(ptr_Product, ldm);
+  naive.load(accumulators);
+
+  // Store the accumulators
+  epilogue.epilogue(accumulators);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <
+  typename ScalarC,
+  /// Specifies the delta between warp accesses along the outer dimension
+  typename WarpDelta
+>
+struct Volta884EpilogueTestbed {
+
+  //
+  // Type definitions
+  //
+
+  /// Warp-level tile
+  typedef cutlass::Shape<4, 64, 64> WarpGemmTile;
+
+  /// Thread-block scoped tile
+  typedef typename cutlass::ShapeMul<
+    WarpGemmTile,
+    WarpDelta
+  >::Shape OutputTile;
+
+  /// Multiply-add operation
+  typedef cutlass::gemm::Volta884MultiplyAdd<
+    WarpGemmTile,
+    cutlass::MatrixLayout::kColumnMajor,
+    half,
+    cutlass::MatrixLayout::kRowMajor,
+    half,
+    ScalarC
+  > MultiplyAdd;
+
+  //
+  // Parameters for the epilogue
+  //
+
+  /// Epilogue functor
+  typedef cutlass::gemm::LinearScaling<ScalarC> Functor;
+
+  /// Traits for global tile access
+  typedef cutlass::gemm::Volta884EpilogueGlobalTileTraits<
+    WarpGemmTile,
+    WarpDelta,
+    1,
+    ScalarC
+  > EpilogueGlobalTileTraits;
+
+
+  /// Defines traits for an epilogue of a Volta884 GEMM
+  typedef cutlass::gemm::Volta884EpilogueTraits<
+    OutputTile,
+    WarpGemmTile,
+    WarpDelta,
+    typename MultiplyAdd::Accumulators,
+    cutlass::gemm::Volta884SelectAccumulators<
+      WarpGemmTile,
+      WarpDelta,
+      ScalarC
+    >,
+    cutlass::PredicatedTileLoadStream<
+      cutlass::TileLoadIterator<
+        EpilogueGlobalTileTraits,
+        ScalarC,
+        cutlass::IteratorAdvance::kH,
+        cutlass::MemorySpace::kGlobal
+      >,
+      cutlass::gemm::Volta884EpiloguePredicateFunctor<EpilogueGlobalTileTraits, ScalarC>
+    >,
+    cutlass::PredicatedTileStoreStream<
+      cutlass::TileStoreIterator<
+        EpilogueGlobalTileTraits,
+        ScalarC,
+        cutlass::IteratorAdvance::kH,
+        cutlass::MemorySpace::kGlobal
+      >,
+      cutlass::gemm::Volta884EpiloguePredicateFunctor<EpilogueGlobalTileTraits, ScalarC>
+    >,
+    cutlass::TileStoreStream<
+      cutlass::gemm::Volta884EpilogueSharedStoreIterator<
+        WarpGemmTile,
+        WarpDelta,
+        ScalarC,
+        ScalarC
+      >
+    >,
+    cutlass::TileLoadStream<
+      cutlass::gemm::Volta884EpilogueSharedLoadIterator<
+        WarpGemmTile,
+        WarpDelta,
+        ScalarC,
+        1,
+        ScalarC
+      >
+    >,
+    Functor
+  > EpilogueTraits;
+
+  //
+  //
+  //
+
+  /// Generates random elements
+  template <typename T>
+  struct RandomGenerator {
+    RandomGenerator(
+      int seed = -1
+    ) { srand(seed); }
+
+    T operator()() {
+      int val = (rand() % 29) - 13;
+      return T(val);
+    }
+  };
+
+  typedef typename cutlass::TypeTraits<ScalarC>::host_type ScalarCHost;
+
+  //
+  // Data members
+  //
+
+  /// Input accumulator matrix
+  cutlass::HostMatrix<ScalarCHost> tensor_C;
+
+  /// Matrix product
+  cutlass::HostMatrix<ScalarCHost> tensor_Product;
+
+  /// Reference output
+  cutlass::HostMatrix<ScalarCHost> tensor_Ref;
+
+  /// Computed output
+  cutlass::HostMatrix<ScalarCHost> tensor_D;
+
+  //
+  // Methods
+  //
+
+  Volta884EpilogueTestbed() {
+    tensor_C.resize(OutputTile::kW, OutputTile::kH, cutlass::MatrixLayout::kColumnMajor);
+    tensor_Product.resize(OutputTile::kW, OutputTile::kH, cutlass::MatrixLayout::kColumnMajor);
+    tensor_Ref.resize(OutputTile::kW, OutputTile::kH, cutlass::MatrixLayout::kColumnMajor);
+    tensor_D.resize_matrix(OutputTile::kW, OutputTile::kH, cutlass::MatrixLayout::kColumnMajor);
+  }
+
+  /// Runs a test case
+  bool run() {
+
+    tensor_C.fill_sequential();
+    tensor_Product.fill_random(RandomGenerator<ScalarCHost>(17));
+
+    tensor_D.fill(ScalarCHost(0));
+    tensor_Ref.fill(ScalarCHost(0));
+
+    tensor_C.sync_device();
+    tensor_Product.sync_device();
+    tensor_D.sync_device();
+
+    // run kernel
+    dim3 grid(1, 1);
+    dim3 block(32 * cutlass::ShapeCount<WarpDelta>::kCount, 1, 1);
+
+    typename EpilogueTraits::Params params;
+
+    params.load_stream_c.iterator.initialize(
+      tensor_C.device_data(),
+      tensor_C.leading_dim(),
+      tensor_C.leading_dim(),
+      1);
+
+    params.store_stream_d.iterator.initialize(
+      tensor_D.device_data(),
+      tensor_D.leading_dim(),
+      tensor_D.leading_dim(),
+      1);
+
+    ScalarCHost alpha = 2;
+    ScalarCHost beta = 1;
+
+    params.functor.initialize(alpha, beta);
+
+    cutlass::Coord<3> problem_size = cutlass::make_Coord(
+        128,
+        64 * EpilogueTraits::WarpDelta::kH - 7,
+        64 * EpilogueTraits::WarpDelta::kW - 5);
+
+    test_volta884_epilogue<EpilogueTraits, ScalarC><<< grid, block >>>(
+      params,
+      tensor_Product.device_data(),
+      tensor_Product.leading_dim(),
+      problem_size
+    );
+
+    EXPECT_EQ(cudaDeviceSynchronize(), cudaSuccess);
+
+    // Copy to host
+    tensor_D.sync_host();
+
+    // Compute reference based on alpha, beta, and the problem dimensions
+    for (int j = 0; j < OutputTile::kH; ++j) {
+      for (int i = 0; i < OutputTile::kW; ++i) {
+        if (j < problem_size[1] && i < problem_size[2]) {
+          tensor_Ref.host_data()[i + j * tensor_Ref.leading_dim()] =
+            alpha * tensor_Product.host_data()[i + j * tensor_Product.leading_dim()] +
+            beta * tensor_C.host_data()[i + j * tensor_C.leading_dim()];
+        }
+      }
+    }
+
+    // Verify result
+    bool passed = tensor_D.bit_equals(tensor_Ref);
+
+    if (!passed) {
+      std::cout << "Mismatch:\n"
+        << "Product = \n" << tensor_Product << "\n\n"
+        << "C =\n" << tensor_C << "\n\n"
+        << "Reference =\n" << tensor_Ref << "\n\n"
+        << "D =\n" << tensor_D << std::endl;
+    }
+
+    return passed;
+  }
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(volta884_epilogue_f32, 64x64x32) {
+
+  Volta884EpilogueTestbed<
+    float,
+    cutlass::Shape<1, 1, 1, 1>
+  > testbed;
+
+  EXPECT_TRUE(testbed.run());
+}
+
+
+TEST(volta884_epilogue_f32, 64x128x32) {
+
+  Volta884EpilogueTestbed<
+    float,
+    cutlass::Shape<1, 2, 1, 1>
+  > testbed;
+
+  EXPECT_TRUE(testbed.run());
+}
+
+
+TEST(volta884_epilogue_f32, 128x64x32) {
+
+  Volta884EpilogueTestbed<
+    float,
+    cutlass::Shape<1, 1, 2, 1>
+  > testbed;
+
+  EXPECT_TRUE(testbed.run());
+}
+
+TEST(volta884_epilogue_f32, 128x128x32) {
+
+  Volta884EpilogueTestbed<
+    float,
+    cutlass::Shape<1, 2, 2, 1>
+  > testbed;
+
+  EXPECT_TRUE(testbed.run());
+}
+
+
+TEST(volta884_epilogue_f32, 256x128x32) {
+
+  Volta884EpilogueTestbed<
+    float,
+    cutlass::Shape<1, 2, 4, 1>
+  > testbed;
+
+  EXPECT_TRUE(testbed.run());
+}
+
+TEST(volta884_epilogue_f32, 128x256x32) {
+
+  Volta884EpilogueTestbed<
+    float,
+    cutlass::Shape<1, 4, 2, 1>
+  > testbed;
+
+  EXPECT_TRUE(testbed.run());
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(volta884_epilogue_f16, 64x64x32) {
+
+  Volta884EpilogueTestbed<
+    half,
+    cutlass::Shape<1, 1, 1, 1>
+  > testbed;
+
+  EXPECT_TRUE(testbed.run());
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(volta884_epilogue_f16, 128x64x32) {
+
+  Volta884EpilogueTestbed<
+    half,
+    cutlass::Shape<1, 1, 2, 1>
+  > testbed;
+
+  EXPECT_TRUE(testbed.run());
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(volta884_epilogue_f16, 64x128x32) {
+
+  Volta884EpilogueTestbed<
+    half,
+    cutlass::Shape<1, 2, 1, 1>
+  > testbed;
+
+  EXPECT_TRUE(testbed.run());
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(volta884_epilogue_f16, 128x128x32) {
+
+  Volta884EpilogueTestbed<
+    half,
+    cutlass::Shape<1, 2, 2, 1>
+  > testbed;
+
+  EXPECT_TRUE(testbed.run());
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(volta884_epilogue_f16, 256x128x32) {
+
+  Volta884EpilogueTestbed<
+    half,
+    cutlass::Shape<1, 2, 4, 1>
+  > testbed;
+
+  EXPECT_TRUE(testbed.run());
+}
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(volta884_epilogue_f16, 128x256x32) {
+
+  Volta884EpilogueTestbed<
+    half,
+    cutlass::Shape<1, 4, 2, 1>
+  > testbed;
+
+  EXPECT_TRUE(testbed.run());
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#endif // CUTLASS_ENABLE_TENSOR_CORE_MMA
+
+// clang-format on

tools/test/unit/gemm/volta884_gemm_threadblock_swizzle.cu

@@ -0,0 +1,496 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+#include <cublas_v2.h>
+#include <cstring>
+#include "cutlass_unit_test.h"
+
+#include "tools/util/half.h"
+#include "tools/util/host_tensor.h"
+#include "tools/util/tensor_view_io.h"
+
+#include "cutlass/gemm/volta884_gemm_traits.h"
+#include "cutlass/gemm/gemm.h"
+#include "cutlass/gemm/threadblock_swizzle.h"
+#include "cutlass/gemm/linear_scaling.h"
+
+#include "tools/test/unit/gemm/gemm_testbed.h"
+#include "tools/test/unit/gemm/run_gemm.h"
+
+#if CUTLASS_ENABLE_TENSOR_CORE_MMA
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+//
+// Very small warp sizes
+//
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_f16_s884gemm_64x64x32_32x32x32_nn_swizzle, short_480x280x224_rowMajorSwizzle) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 32, 32>,
+    float,
+    float,
+    float,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+TEST(Volta884_f16_s884gemm_64x64x32_32x32x32_nn_swizzle, short_480x280x224_rowMajorSwizzle_groupCol2) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 32, 32>,
+    float,
+    float,
+    float,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+TEST(Volta884_f16_s884gemm_64x64x32_32x32x32_nn_swizzle, short_480x280x224_rowMajorSwizzle_Boustrophedon) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 32, 32>,
+    float,
+    float,
+    float,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+TEST(Volta884_f16_s884gemm_64x64x32_32x32x32_nn_swizzle, short_480x280x224_rowMajorSwizzle_groupCol2_Boustrophedon) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 32, 32>,
+    float,
+    float,
+    float,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_f16_s884gemm_64x64x32_32x32x32_nn_swizzle, short_480x280x224_columnMajorSwizzle) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 32, 32>,
+    float,
+    float,
+    float,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+TEST(Volta884_f16_s884gemm_64x64x32_32x32x32_nn_swizzle, short_480x280x224_columnMajorSwizzle_groupCol2) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 32, 32>,
+    float,
+    float,
+    float,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+TEST(Volta884_f16_s884gemm_64x64x32_32x32x32_nn_swizzle, short_480x280x224_columnMajorSwizzle_Boustrophedon) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 32, 32>,
+    float,
+    float,
+    float,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+TEST(Volta884_f16_s884gemm_64x64x32_32x32x32_nn_swizzle, short_480x280x224_columnMajorSwizzle_groupCol2_Boustrophedon) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 32, 32>,
+    float,
+    float,
+    float,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_f16_s884gemm_64x64x32_32x32x32_tt_swizzle, short_480x280x224_rowMajorSwizzle) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 32, 32>,
+    float,
+    float,
+    float,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+TEST(Volta884_f16_s884gemm_64x64x32_32x32x32_tt_swizzle, short_480x280x224_rowMajorSwizzle_groupCol2) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 32, 32>,
+    float,
+    float,
+    float,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+TEST(Volta884_f16_s884gemm_64x64x32_32x32x32_tt_swizzle, short_480x280x224_rowMajorSwizzle_Boustrophedon) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 32, 32>,
+    float,
+    float,
+    float,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+TEST(Volta884_f16_s884gemm_64x64x32_32x32x32_tt_swizzle, short_480x280x224_rowMajorSwizzle_groupCol2_Boustrophedon) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 32, 32>,
+    float,
+    float,
+    float,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_f16_s884gemm_64x64x32_32x32x32_tt_swizzle, short_480x280x224_columnMajorSwizzle) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 32, 32>,
+    float,
+    float,
+    float,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+TEST(Volta884_f16_s884gemm_64x64x32_32x32x32_tt_swizzle, short_480x280x224_columnMajorSwizzle_groupCol2) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 32, 32>,
+    float,
+    float,
+    float,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+TEST(Volta884_f16_s884gemm_64x64x32_32x32x32_tt_swizzle, short_480x280x224_columnMajorSwizzle_Boustrophedon) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 32, 32>,
+    float,
+    float,
+    float,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+TEST(Volta884_f16_s884gemm_64x64x32_32x32x32_tt_swizzle, short_480x280x224_columnMajorSwizzle_groupCol2_Boustrophedon) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::MatrixLayout::kRowMajor,
+    cutlass::Shape<32, 64, 64>,
+    cutlass::Shape<32, 32, 32>,
+    float,
+    float,
+    float,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+//
+// FP32 accumulation, FP16 output
+//
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_f16_s884gemm_f16_128x128x32_nn_swizzle, 480x280x224_rowMajorSwizzle) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 128, 128>,
+    cutlass::Shape<32, 64, 64>,
+    float,
+    half,
+    half,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+TEST(Volta884_f16_s884gemm_f16_128x128x32_nn_swizzle, 480x280x224_rowMajorSwizzle_groupCol2) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 128, 128>,
+    cutlass::Shape<32, 64, 64>,
+    float,
+    half,
+    half,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+TEST(Volta884_f16_s884gemm_f16_128x128x32_nn_swizzle, 480x280x224_rowMajorSwizzle_Boustrophedon) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 128, 128>,
+    cutlass::Shape<32, 64, 64>,
+    float,
+    half,
+    half,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+TEST(Volta884_f16_s884gemm_f16_128x128x32_nn_swizzle, 480x280x224_rowMajorSwizzle_groupCol2_Boustrophedon) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 128, 128>,
+    cutlass::Shape<32, 64, 64>,
+    float,
+    half,
+    half,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Volta884_f16_s884gemm_f16_128x128x32_nn_swizzle, 480x280x224_columnMajorSwizzle) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 128, 128>,
+    cutlass::Shape<32, 64, 64>,
+    float,
+    half,
+    half,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+TEST(Volta884_f16_s884gemm_f16_128x128x32_nn_swizzle, 480x280x224_columnMajorSwizzle_groupCol2) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 128, 128>,
+    cutlass::Shape<32, 64, 64>,
+    float,
+    half,
+    half,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+TEST(Volta884_f16_s884gemm_f16_128x128x32_nn_swizzle, 480x280x224_columnMajorSwizzle_Boustrophedon) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 128, 128>,
+    cutlass::Shape<32, 64, 64>,
+    float,
+    half,
+    half,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+TEST(Volta884_f16_s884gemm_f16_128x128x32_nn_swizzle, 480x280x224_columnMajorSwizzle_groupCol2_Boustrophedon) {
+
+  typedef cutlass::gemm::Volta884GemmTraits<
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::MatrixLayout::kColumnMajor,
+    cutlass::Shape<32, 128, 128>,
+    cutlass::Shape<32, 64, 64>,
+    float,
+    half,
+    half,
+    2,
+    cutlass::gemm::LinearScaling<float>,
+    typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>
+  > GemmTraits;
+
+  run_gemm<GemmTraits>(480, 280, 224);
+}
+
+
+#endif // if defined(CUTLASS_ENABLE_TENSOR_CORE_MMA)