Cutlass 1.3 Release (#42)

CUTLASS 1.3 Release
- Efficient GEMM kernel targeting Volta Tensor Cores via mma.sync instruction added in CUDA 10.1.

tools/test/perf/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification, are permitted
 # provided that the following conditions are met:
@@ -32,6 +32,8 @@ set(CUTLASS_PERF_TEST_HEADERS
   gemm/cutlass_dispatch_splitK_PI.h
   gemm/gemm_perf_testbed.h
   gemm/gemm_profiler.h
+  gemm/cutlass_volta884_dispatch.h
+  gemm/cutlass_volta884_dispatch_splitK_PI.h
 )
 
 set(CUTLASS_PERF_TEST_SOURCES
@@ -43,8 +45,14 @@ set(CUTLASS_PERF_TEST_SOURCES
   gemm/igemm.cu
   gemm/igemm_splitK.cu
   gemm/wmma_gemm.cu
-  gemm/wmma_binary_gemm.cu
-  gemm/wmma_integer_gemm.cu
+  gemm/volta884_gemm.cu                       
+  gemm/volta884_gemm_splitK.cu
+  gemm/volta884_gemm_cta_rasterization_tn.cu
+  gemm/volta884_gemm_cta_rasterization_tt.cu
+  gemm/volta884_gemm_cta_rasterization_nn.cu
+  gemm/volta884_gemm_cta_rasterization_nt.cu
+  gemm/wmma_binary_gemm.cu                         
+  gemm/wmma_integer_gemm.cu                        
 )
 
 source_group("Source\ Files" FILES ${CUTLASS_PERF_TEST_SOURCES})
@@ -62,5 +70,7 @@ cutlass_add_executable(
     ${CUTLASS_PERF_TEST_HEADERS}
 )
 
-target_link_libraries(cutlass_perf_test ${CUBLAS_LIBRARY})
+if(CUTLASS_ENABLE_CUBLAS)
+  target_link_libraries(cutlass_perf_test ${CUBLAS_LIBRARY})
+endif()
 

tools/test/perf/cutlass_perf_test.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/perf/cutlass_perf_test.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -32,6 +32,14 @@
 #include "tools/test/perf/testbench_output.h"
 #include "tools/test/perf/gemm/gemm_profiler.h"
 
+#if !defined(CUTLASS_ENABLE_CUBLAS)
+#define CUTLASS_ENABLE_CUBLAS 0
+#endif
+
+#if !defined(CUTLASS_ENABLE_CUDNN)
+#define CUTLASS_ENABLE_CUDNN 0
+#endif
+
 namespace perf {
 
 typedef int (GemmProfileFunc)(

tools/test/perf/gemm/cublas_dispatch.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -67,6 +67,7 @@ struct CublasGemmDispatch {
                             CDeviceType *C,
                             int ldc,
                             cublasGemmAlgo_t algorithm) {
+    #if CUTLASS_ENABLE_CUBLAS
     return cublasGemmEx(handle,
                         convert(layout_a),
                         convert(layout_b),
@@ -86,6 +87,9 @@ struct CublasGemmDispatch {
                         ldc,
                         cutlass::TypeTraits<AccumulatorDeviceType>::cublas_type,
                         algorithm);
+    #else
+    return CUBLAS_STATUS_NOT_SUPPORTED;
+    #endif
   }
 };
 
@@ -131,7 +135,7 @@ struct CublasBatchedStridedGemmDispatch {
     long long int batch_stride_C,
     int batch_count,
     cublasGemmAlgo_t algorithm) {
-#if defined(CUDA_VERSION) && CUDA_VERSION >= 9010
+    #if CUTLASS_ENABLE_CUBLAS && defined(CUDA_VERSION) && CUDA_VERSION >= 9010
     return cublasGemmStridedBatchedEx(handle,
       convert(layout_a),
       convert(layout_b),
@@ -155,9 +159,9 @@ struct CublasBatchedStridedGemmDispatch {
       batch_count,
       cutlass::TypeTraits<AccumulatorDeviceType>::cublas_type,
       algorithm);
-#else
+    #else
     return CUBLAS_STATUS_NOT_SUPPORTED;
-#endif
+    #endif
   }
 };
 

tools/test/perf/gemm/cutlass_dispatch.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -33,7 +33,12 @@ template <typename Gemm_,
           typename Compute_,
           typename ScalarEpilogue_,
           bool ThreadMultiplyAdd_,
-          bool RunCuBLAS_ = true>
+          #if CUTLASS_ENABLE_CUBLAS
+          bool RunCuBLAS_ = true
+          #else
+          bool RunCuBLAS_ = false
+          #endif
+>
 struct CutlassDispatch {
   typedef typename Gemm_::Params Params;
   typedef Gemm_ Gemm;
@@ -131,8 +136,6 @@ struct CutlassDispatchBasic {
   typedef typename Traits::ScalarC ScalarC;
   /// The scalar for D.
   typedef typename Traits::ScalarD ScalarD;
-
-  // TODO - support alternative accumulator and scalar types
   typedef ScalarD Compute;
   typedef Compute ScalarEpilogue;
 

tools/test/perf/gemm/cutlass_dispatch_splitK_PI.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
-* Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+* Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification, are permitted
 * provided that the following conditions are met:
@@ -40,8 +40,13 @@ template <typename KernelClass_,
     typename Compute_,
     typename ScalarEpilogue_,
     bool ThreadMultiplyAdd_,
-    bool RunCuBLAS_ = true>
-  struct CutlassDispatchSplitKPIGemm {
+    #if CUTLASS_ENABLE_CUBLAS
+    bool RunCuBLAS_ = true
+    #else
+    bool RunCuBLAS_ = false
+    #endif
+>
+struct CutlassDispatchSplitKPIGemm {
   typedef typename KernelClass_::Params Params;
   typedef KernelClass_ KernelClass;
   typedef Index_ Index;
@@ -87,8 +92,21 @@ template <typename KernelClass_,
     ScalarD* d_d,
     Index ldd) {
     params.init_problem(m, n, k);
-    int workspace_size_in_byte = params.required_workspace_memory_in_byte();
-    
+    size_t workspace_size_in_byte = params.required_workspace_memory_in_byte();
+    size_t available_device_memory_in_byte = 0;
+    size_t device_memory_in_byte = 0;
+    cudaError_t cudaMemGetInfo_err = cudaMemGetInfo(&available_device_memory_in_byte, &device_memory_in_byte);
+    if (cudaMemGetInfo_err != cudaSuccess) {
+      std::cout << "\ncudaMemGetInfo error: " << cudaGetErrorString(cudaMemGetInfo_err)
+        << "\n";
+    }
+
+    if (workspace_size_in_byte > available_device_memory_in_byte) {
+      std::cout << "reqested workspace memory size("<< workspace_size_in_byte << 
+                   ") is larger than available memory size("<< available_device_memory_in_byte << "). Abort." << std::endl;
+      throw std::runtime_error("reqested workspace memory size is larger than available memory size. Abort.");
+    }
+
     cudaError_t workspace_err = cudaMalloc(&workspace_ptr, workspace_size_in_byte);
     if (workspace_err != cudaSuccess) {
       std::cout << "\nCUDA workspace malloc error: " << cudaGetErrorString(workspace_err)
@@ -153,8 +171,6 @@ struct CutlassDispatchSplitKPIGemmBasic {
   typedef typename Traits::ScalarC ScalarC;
   /// The scalar for D.
   typedef typename Traits::ScalarD ScalarD;
-
-  // TODO - support alternative accumulator and scalar types
   typedef ScalarD Compute;
   typedef Compute ScalarEpilogue;
 

tools/test/perf/gemm/cutlass_volta884_dispatch.h

@@ -0,0 +1,113 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma once
+
+
+#include "cutlass/gemm/gemm.h"
+#include "cutlass/gemm/volta884_gemm_traits.h"
+
+#include "tools/test/perf/cutlass_perf_test.h"
+#include "tools/test/perf/gemm/gemm_profiler.h"
+#include "tools/test/perf/gemm/cutlass_dispatch.h"
+#include "tools/test/perf/gemm/gemm_perf_testbed.h"
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename Traits>
+struct Volta884GemmDispatch {
+
+  typedef cutlass::gemm::Gemm<Traits> Gemm;
+
+  typedef typename Gemm::Params Params;
+
+  typedef typename Traits::ScalarC ScalarC;
+  typedef typename Traits::ScalarD ScalarD;
+  typedef typename Traits::ScalarD ScalarEpilogue;
+
+  /// Indicate warp-level GEMM
+  static bool const kThreadMultiplyAdd = false;
+
+  #if CUTLASS_ENABLE_CUBLAS
+  static bool const kRunCuBLAS = true;
+  #else
+  static bool const kRunCuBLAS = false;
+  #endif
+
+  static cutlass::MatrixLayout::Kind const kLayoutA = Traits::kLayoutA;
+  static cutlass::MatrixLayout::Kind const kLayoutB = Traits::kLayoutB;
+
+  //
+  // Data members
+  //
+
+  /// Params argument
+  Params params;
+
+  //
+  // Methods
+  //
+
+  Volta884GemmDispatch() {}
+
+  /// Initializes params object
+  Volta884GemmDispatch(int m, int n, int k, ScalarEpilogue alpha, half const* d_a, int lda,
+                  half const* d_b, int ldb, ScalarEpilogue beta, ScalarC const* d_c, int ldc,
+                  ScalarD* d_d, int ldd) {
+
+    params.initialize(m, n, k, alpha, d_a, lda, d_b, ldb, beta, d_c, ldc, d_d, ldd);
+  }
+  
+  Volta884GemmDispatch(int m,
+    int n,
+    int k,
+    ScalarEpilogue alpha,
+    half const* d_a,
+    int lda,
+    long long int batch_stride_A,
+    half const* d_b,
+    int ldb,
+    long long int batch_stride_B,
+    ScalarEpilogue beta,
+    ScalarC const* d_c,
+    int ldc,
+    long long int batch_stride_C,
+    ScalarD* d_d,
+    int ldd,
+    long long int batch_stride_D,
+    int batch_count) {
+    assert(0);//not yet supported
+  }
+
+  /// Initializes params object
+  Volta884GemmDispatch(Params const& _params) : params(_params) {}
+
+  /// Launches kernel
+  cudaError_t operator()() { return Gemm::launch(params); }
+};

tools/test/perf/gemm/cutlass_volta884_dispatch_splitK_PI.h

@@ -0,0 +1,139 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma once
+
+
+#include "cutlass/gemm/device_gemm.h"
+#include "cutlass/gemm/volta884_gemm_traits.h"
+
+#include "tools/test/perf/cutlass_perf_test.h"
+#include "tools/test/perf/gemm/gemm_profiler.h"
+#include "tools/test/perf/gemm/cutlass_dispatch.h"
+#include "tools/test/perf/gemm/gemm_perf_testbed.h"
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename Traits>
+struct Volta884GemmDispatchSplitKPI {
+
+  typedef cutlass::gemm::DeviceGemm<Traits> Gemm;
+
+  typedef typename Gemm::Params Params;
+
+  typedef typename Traits::ScalarC ScalarC;
+  typedef typename Traits::ScalarD ScalarD;
+  typedef typename Traits::Scalar ScalarEpilogue;
+
+  /// Indicate warp-level GEMM
+  static bool const kThreadMultiplyAdd = false;
+
+  #if CUTLASS_ENABLE_CUBLAS
+  static bool const kRunCuBLAS = true;
+  #else
+  static bool const kRunCuBLAS = false;
+  #endif
+
+  static cutlass::MatrixLayout::Kind const kLayoutA = Traits::kLayoutA;
+  static cutlass::MatrixLayout::Kind const kLayoutB = Traits::kLayoutB;
+
+  //
+  // Data members
+  //
+
+  /// Params argument
+  Params params;
+
+  /// splitK PI require workspace
+  typename cutlass::TypeTraits<typename Traits::ScalarAccum>::device_type *workspace_ptr;
+
+  //
+  // Methods
+  //
+
+  Volta884GemmDispatchSplitKPI() {}
+
+  /// Initializes params object
+  Volta884GemmDispatchSplitKPI(int m, int n, int k, ScalarEpilogue alpha, half const* d_a, int lda,
+                  half const* d_b, int ldb, ScalarEpilogue beta, ScalarC const* d_c, int ldc,
+                  ScalarD* d_d, int ldd) {
+    params.init_problem(m, n, k);
+    size_t workspace_size_in_byte = params.required_workspace_memory_in_byte();
+    size_t available_device_memory_in_byte = 0;
+    size_t device_memory_in_byte = 0;
+    cudaError_t cudaMemGetInfo_err = cudaMemGetInfo(&available_device_memory_in_byte, &device_memory_in_byte);
+    if (cudaMemGetInfo_err != cudaSuccess) {
+      std::cout << "\ncudaMemGetInfo error: " << cudaGetErrorString(cudaMemGetInfo_err)
+        << "\n";
+    }
+
+    if (workspace_size_in_byte > available_device_memory_in_byte) {
+      std::cout << "reqested workspace memory size(" << workspace_size_in_byte <<
+        ") is larger than available memory size(" << available_device_memory_in_byte << "). Abort." << std::endl;
+      throw std::runtime_error("reqested workspace memory size is larger than available memory size. Abort.");
+    }
+
+    cudaError_t workspace_err = cudaMalloc(&workspace_ptr, workspace_size_in_byte);
+    if (workspace_err != cudaSuccess) {
+      std::cout << "\nCUDA workspace malloc error: " << cudaGetErrorString(workspace_err)
+        << "\n";
+    }
+
+    params.initialize(alpha, d_a, lda, d_b, ldb, beta, d_c, ldc, d_d, ldd, workspace_ptr, 8 /*volta884 requires leading dim to be mulitiple of 8*/);
+  }
+
+  Volta884GemmDispatchSplitKPI(int m,
+    int n,
+    int k,
+    ScalarEpilogue alpha,
+    half const* d_a,
+    int lda,
+    long long int batch_stride_A,
+    half const* d_b,
+    int ldb,
+    long long int batch_stride_B,
+    ScalarEpilogue beta,
+    ScalarC const* d_c,
+    int ldc,
+    long long int batch_stride_C,
+    ScalarD* d_d,
+    int ldd,
+    long long int batch_stride_D,
+    int batch_count) {
+    assert(0);//not yet supported
+  }
+
+  /// Initializes params object
+  Volta884GemmDispatchSplitKPI(Params const& _params) : params(_params) {}
+
+  /// Launches kernel
+  cudaError_t operator()() { 
+    return Gemm::launch(params); 
+  }
+};

tools/test/perf/gemm/dgemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/perf/gemm/gemm_perf_testbed.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -284,10 +284,14 @@ class GemmTestbed {
   /// Constructs a basic workspace
   GemmTestbed(InitialDistribution const &_dist = InitialDistribution())
       : initial_distribution(_dist) {
+    #if CUTLASS_ENABLE_CUBLAS
     status = cublasCreate(&handle);
     if (status != CUBLAS_STATUS_SUCCESS) {
       throw cutlass::cuda_exception("Failed to create CUBLAS handle");
     }
+    #else
+    status = CUBLAS_STATUS_NOT_INITIALIZED;
+    #endif
   }
 
   /// Constructs a workspace for verifying GEMM, assumes
@@ -296,15 +300,26 @@ class GemmTestbed {
               cublasGemmAlgo_t algorithm_ = CUBLAS_GEMM_DEFAULT,
               InitialDistribution const &_dist = InitialDistribution())
       : problem(_problem), initial_distribution(_dist) {
+    #if CUTLASS_ENABLE_CUBLAS
     status = cublasCreate(&handle);
     if (status != CUBLAS_STATUS_SUCCESS) {
       throw cutlass::cuda_exception("Failed to create CUBLAS handle");
     }
+    #else
+    status = CUBLAS_STATUS_NOT_INITIALIZED;
+    #endif
 
     resize(problem);
   }
 
-  ~GemmTestbed() { status = cublasDestroy(handle); }
+  /// Destructs the GEMM testbed
+  ~GemmTestbed() {
+    #if CUTLASS_ENABLE_CUBLAS
+    if (status != CUBLAS_STATUS_NOT_INITIALIZED) {
+      status = cublasDestroy(handle);
+    }
+    #endif
+  }
 
   /// Returns true if the last CUBLAS call returned successfully
   bool good() const { return status == CUBLAS_STATUS_SUCCESS; }
@@ -388,6 +403,7 @@ class GemmTestbed {
 
   /// Launches the cuBLAS GEMM - does not initialize output matrix
   cublasStatus_t launch_cublas(cublasGemmAlgo_t algo) {
+    #if CUTLASS_ENABLE_CUBLAS
     if (problem.batch_count == 1) {
       CublasDispatch dispatch;
 
@@ -441,6 +457,9 @@ class GemmTestbed {
 
       return status;
     }
+    #else
+    return CUBLAS_STATUS_NOT_SUPPORTED;
+    #endif
   }
 
   /// Verifies the 'test' tensor with 'ref'

tools/test/perf/gemm/gemm_profiler.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -146,22 +146,23 @@ class GemmProfiler {
       , kernel_name
       , problem
     );
-
+    
+    result.disposition = Disposition::NotVerified;
+    
     if (options.dry_run) {
       result.disposition = Disposition::NotRun;
       return result;
     }
 
     if (CutlassDispatch::kRunCuBLAS) {
+#if CUTLASS_ENABLE_CUBLAS
       testbed.compute_reference(algorithm);
 
       if (cudaDeviceSynchronize() != cudaSuccess) {
         result.disposition = Disposition::NotVerified;
         return result;
       }
-    }
-    else {
-      result.disposition = Disposition::Passed;
+#endif
     }
 
     CutlassDispatch *dispatch_ptr;
@@ -214,11 +215,13 @@ class GemmProfiler {
     }
 
     if (CutlassDispatch::kRunCuBLAS) {
+#if CUTLASS_ENABLE_CUBLAS
       if (testbed.verify_with_reference()) {
         result.disposition = Disposition::Passed;
       } else {
         result.disposition = Disposition::Incorrect;
       }
+#endif
     }
 
     if (options.save_workspace(result.disposition == Disposition::Passed)) {
@@ -270,11 +273,34 @@ class GemmProfiler {
     result.runtime = double(average_ms) / double(options.iterations);
     result.gflops = testbed.GFLOPs_per_sec(result.runtime);
 
-    if (result.disposition != Disposition::Passed) {
-      std::cout << "[\033[1;31mFAILED\033[0m]: " << kernel_name
-                << " failed with disposition: " << result.disposition << "\n";
+    if (result.disposition == Disposition::Unknown) {
+      std::cout << "[\033[1;30mUnknown\033[0m]: " << kernel_name
+                << " with disposition: " << result.disposition << "\n";
+    }
+    if (result.disposition == Disposition::NotRun) {
+      std::cout << "[\033[1;33mNotRun\033[0m]: " << kernel_name
+                << " with disposition: " << result.disposition << "\n";
+    }
+    if (result.disposition == Disposition::Passed) {
+      std::cout << "[\033[1;32mPassed\033[0m]: " << kernel_name
+                << " with disposition: " << result.disposition << "\n";
+    }
+    if (result.disposition == Disposition::Incorrect) {
+      std::cout << "[\033[1;31mIncorrect\033[0m]: " << kernel_name
+                << " with disposition: " << result.disposition << "\n";
+    }
+    if (result.disposition == Disposition::Failed) {
+      std::cout << "[\033[1;31mFailed\033[0m]: " << kernel_name
+                << " with disposition: " << result.disposition << "\n";
+    }
+    if (result.disposition == Disposition::NotVerified) {
+      std::cout << "[\033[1;34mNotVerified\033[0m]: " << kernel_name
+                << " with disposition: " << result.disposition << "\n";
+    }
+    if (result.disposition == Disposition::Invalid) {
+      std::cout << "[\033[1;36mInvalid\033[0m]: " << kernel_name
+                << " with disposition: " << result.disposition << "\n";
     }
-
     delete dispatch_ptr;
     return result;
   }
@@ -299,7 +325,7 @@ class GemmProfiler {
 
     std::vector<PerformanceResult<GemmProblem> > results;
 
-    results.push_back(execute_cutlass<CutlassDispatch>(problem, algorithm));
+      results.push_back(execute_cutlass<CutlassDispatch>(problem, algorithm));
     // cool-down period
     if (!options.dry_run) {
       pause(options.sleep_time);
@@ -402,10 +428,10 @@ int profile_gemm(TestbenchOutput<GemmProblem> &output,
     GemmProfiler perf(output, kernel, cutlass_algo, options, config);
     if (options.peak_performance) {
       perf.template peak<Dispatch>(
-          config.problem_range.M, config.problem_range.N, config.problem_range.K);
+          config.gemm_problem_range.M, config.gemm_problem_range.N, config.gemm_problem_range.K);
     } else {
       perf.template schmoo<Dispatch>(
-          config.problem_range.M, config.problem_range.N, config.problem_range.K, config.problem_range.batch_count);
+          config.gemm_problem_range.M, config.gemm_problem_range.N, config.gemm_problem_range.K, config.gemm_problem_range.batch_count);
     }
   }
 

tools/test/perf/gemm/hgemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/perf/gemm/igemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -23,6 +23,8 @@
  *
  **************************************************************************************************/
 
+#if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))
+
 #include "cutlass/gemm/gemm.h"
 #include "cutlass/gemm/igemm_traits.h"
 #include "tools/test/perf/cutlass_perf_test.h"
@@ -36,6 +38,7 @@ namespace perf {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
+template <typename DummyT>
 int profile_igemm(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
 
   typedef perf::GemmProfiler<int8_t, int8_t, int, int, int> GemmProfiler;
@@ -91,6 +94,21 @@ int profile_igemm(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &
     results |= profile_gemm<Dispatch, GemmProfiler>(output, "igemm_tt", options, config);
   }
 
+  return results;
+}
+
+template <typename DummyT>
+int profile_igemm_32x32x128(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
+
+  typedef perf::GemmProfiler<int8_t, int8_t, int, int, int> GemmProfiler;
+
+  // compute capability check
+  if (!options.compute_capability(6, 1)) {
+    return 0;
+  }
+
+  int results = 0;
+
   {
     typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
             cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<128, 32, 32>, int,
@@ -138,8 +156,18 @@ int profile_igemm(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &
   return results;
 }
 
+
+
 struct IgemmRegistrar {
-  IgemmRegistrar() { RegisterGemmProfileFunc(profile_igemm); }
+  IgemmRegistrar()
+  {
+    RegisterGemmProfileFunc(profile_igemm<void>);
+
+#ifdef EXHAUSTIVE_PROF
+    RegisterGemmProfileFunc(profile_igemm_32x32x128<void>);
+#endif // defined EXHAUSTIVE_PROF
+
+  }
 };
 
 volatile IgemmRegistrar _IgemmRegistrar;
@@ -147,3 +175,5 @@ volatile IgemmRegistrar _IgemmRegistrar;
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 } // namespace perf
+
+#endif // if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))

tools/test/perf/gemm/igemm_splitK.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -23,6 +23,8 @@
  *
  **************************************************************************************************/
 
+#if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))
+
 #include "cutlass/gemm/gemm.h"
 #include "cutlass/gemm/igemm_traits.h"
 #include "cutlass/reduction/batched_reduction_traits.h"
@@ -154,7 +156,6 @@ int profile_igemm_splitkpi_kernel(
     results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_tt", options, config, algo + "_splitk_pi");
   }
 
-
   return results;
 }
 
@@ -200,3 +201,5 @@ volatile IgemmSplitKPIRegistrar _IgemmSplitKPIRegistrar;
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 } // namespace perf
+
+#endif // if (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 610))

tools/test/perf/gemm/sgemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -38,8 +38,10 @@ namespace perf {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
+
+/// Profile simple gemm kernels
 template <typename OutputTile>
-int profile_sgemm_kernel(
+int profile_simple_sgemm_kernel(
   TestbenchOutput<GemmProblem> &output,
   TestbenchOptions const &options,
   Config const &config,
@@ -98,6 +100,24 @@ int profile_sgemm_kernel(
     results |= profile_gemm<Dispatch, SGemmProfiler>(output, name + "_tt", options, config, algo);
   }
 
+  return results;
+}
+
+
+
+/// Profile swizzle-raster gemm kernels
+template <typename OutputTile>
+int profile_swizzle_sgemm_kernel(
+  TestbenchOutput<GemmProblem> &output,
+  TestbenchOptions const &options,
+  Config const &config,
+  std::string const &name,
+  std::string const &algo) {
+
+  typedef perf::GemmProfiler<float, float, float, float, float> SGemmProfiler;
+
+  int results = 0;
+
   {
     typedef int index;
     typedef cutlass::gemm::SgemmConfig<OutputTile,
@@ -1321,7 +1341,11 @@ int profile_sgemm_kernel(
 int profile_sgemm(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
   int results = 0;
 
-  results |= profile_sgemm_kernel<cutlass::Shape<8, 128, 128> >(output, options, config, "sgemm", "128x128");
+  results |= profile_simple_sgemm_kernel<cutlass::Shape<8, 128, 128> >(output, options, config, "sgemm", "128x128");
+
+#ifdef EXHAUSTIVE_PROF
+  results |= profile_swizzle_sgemm_kernel<cutlass::Shape<8, 128, 128> >(output, options, config, "sgemm", "128x128");
+#endif // defined EXHAUSTIVE_PROF
 
   return results;
 }

tools/test/perf/gemm/sgemm_splitK.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -38,7 +38,7 @@ namespace perf {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template <typename OutputTile, int splits_count>
+template <typename OutputTile, typename threadGemmShape, typename threadReductionShape, int splits_count>
 int profile_sgemm_splitkpi_kernel(
   TestbenchOutput<GemmProblem> &output,
   TestbenchOptions const &options,
@@ -53,7 +53,8 @@ int profile_sgemm_splitkpi_kernel(
   {
     /*batched sgemm traits*/
     typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
-      cutlass::MatrixLayout::kColumnMajor, OutputTile>
+      cutlass::MatrixLayout::kColumnMajor, OutputTile,
+      cutlass::gemm::LinearScaling<float>, threadGemmShape>
       SgemmTraits;
     /*batched reduction traits*/
     typedef cutlass::reduction::BatchedReductionTraits<float,
@@ -64,7 +65,7 @@ int profile_sgemm_splitkpi_kernel(
       splits_count,
       cutlass::Shape<1, 1, 128>,
       cutlass::Shape<1, 1, 64>,
-      cutlass::Shape<1, 1, 2> >
+      threadReductionShape >
       BatchedReductionTraits;
 
     // create a device gemm 
@@ -77,7 +78,8 @@ int profile_sgemm_splitkpi_kernel(
   {
     /*batched sgemm traits*/
     typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
-      cutlass::MatrixLayout::kRowMajor, OutputTile>
+      cutlass::MatrixLayout::kRowMajor, OutputTile,
+      cutlass::gemm::LinearScaling<float>, threadGemmShape>
       SgemmTraits;
     /*batched reduction traits*/
     typedef cutlass::reduction::BatchedReductionTraits<float,
@@ -88,7 +90,7 @@ int profile_sgemm_splitkpi_kernel(
       splits_count,
       cutlass::Shape<1, 1, 128>,
       cutlass::Shape<1, 1, 64>,
-      cutlass::Shape<1, 1, 2> >
+      threadReductionShape >
       BatchedReductionTraits;
 
     // create a device gemm 
@@ -101,7 +103,8 @@ int profile_sgemm_splitkpi_kernel(
   {
     /*batched sgemm traits*/
     typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
-      cutlass::MatrixLayout::kColumnMajor, OutputTile>
+      cutlass::MatrixLayout::kColumnMajor, OutputTile,
+      cutlass::gemm::LinearScaling<float>, threadGemmShape>
       SgemmTraits;
     /*batched reduction traits*/
     typedef cutlass::reduction::BatchedReductionTraits<float,
@@ -112,7 +115,7 @@ int profile_sgemm_splitkpi_kernel(
       splits_count,
       cutlass::Shape<1, 1, 128>,
       cutlass::Shape<1, 1, 64>,
-      cutlass::Shape<1, 1, 2> >
+      threadReductionShape >
       BatchedReductionTraits;
 
     // create a device gemm 
@@ -125,7 +128,8 @@ int profile_sgemm_splitkpi_kernel(
   {
     /*batched sgemm traits*/
     typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
-      cutlass::MatrixLayout::kRowMajor, OutputTile>
+      cutlass::MatrixLayout::kRowMajor, OutputTile,
+      cutlass::gemm::LinearScaling<float>, threadGemmShape>
       SgemmTraits;
     /*batched reduction traits*/
     typedef cutlass::reduction::BatchedReductionTraits<float,
@@ -136,7 +140,7 @@ int profile_sgemm_splitkpi_kernel(
       splits_count,
       cutlass::Shape<1, 1, 128>,
       cutlass::Shape<1, 1, 64>,
-      cutlass::Shape<1, 1, 2> >
+      threadReductionShape >
       BatchedReductionTraits;
 
     // create a device gemm 
@@ -153,25 +157,143 @@ int profile_sgemm_splitkpi_kernel(
 /// Profiles all SGEMM tile sizes
 int profile_sgemm_splitkpi(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
   int results = 0;
+  /*128x128x8*/
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 5 >(output, options, config, "sgemm_128x128x8_splitk_pi_split5", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 8 >(output, options, config, "sgemm_128x128x8_splitk_pi_split8", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "sgemm_128x128x8_splitk_pi_split10", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 16 >(output, options, config, "sgemm_128x128x8_splitk_pi_split16", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 20 >(output, options, config, "sgemm_128x128x8_splitk_pi_split20", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 24 >(output, options, config, "sgemm_128x128x8_splitk_pi_split24", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 28 >(output, options, config, "sgemm_128x128x8_splitk_pi_split28", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 32 >(output, options, config, "sgemm_128x128x8_splitk_pi_split32", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 40 >(output, options, config, "sgemm_128x128x8_splitk_pi_split40", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 48 >(output, options, config, "sgemm_128x128x8_splitk_pi_split48", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 56 >(output, options, config, "sgemm_128x128x8_splitk_pi_split56", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 64 >(output, options, config, "sgemm_128x128x8_splitk_pi_split64", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 72 >(output, options, config, "sgemm_128x128x8_splitk_pi_split72", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 80 >(output, options, config, "sgemm_128x128x8_splitk_pi_split80", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 88 >(output, options, config, "sgemm_128x128x8_splitk_pi_split88", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 96 >(output, options, config, "sgemm_128x128x8_splitk_pi_split96", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 104 >(output, options, config, "sgemm_128x128x8_splitk_pi_split104", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 112 >(output, options, config, "sgemm_128x128x8_splitk_pi_split112", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 120 >(output, options, config, "sgemm_128x128x8_splitk_pi_split120", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 128 >(output, options, config, "sgemm_128x128x8_splitk_pi_split128", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 136 >(output, options, config, "sgemm_128x128x8_splitk_pi_split136", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 144 >(output, options, config, "sgemm_128x128x8_splitk_pi_split144", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 152 >(output, options, config, "sgemm_128x128x8_splitk_pi_split152", "128x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 160 >(output, options, config, "sgemm_128x128x8_splitk_pi_split160", "128x128");
 
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 128>, 32 >(output, options, config, "sgemm_128x128x8_splitk_pi_split32", "128x128");
-
+#ifdef EXHAUSTIVE_PROF
   /*128x64x8*/
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, 8 >(output, options, config, "sgemm_128x64x8_splitk_pi_split8", "128x64");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, 16 >(output, options, config, "sgemm_128x64x8_splitk_pi_split16", "128x64");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, 20 >(output, options, config, "sgemm_128x64x8_splitk_pi_split20", "128x64");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, 24 >(output, options, config, "sgemm_128x64x8_splitk_pi_split24", "128x64");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, 28 >(output, options, config, "sgemm_128x64x8_splitk_pi_split28", "128x64");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, 32 >(output, options, config, "sgemm_128x64x8_splitk_pi_split32", "128x64");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, 64 >(output, options, config, "sgemm_128x64x8_splitk_pi_split64", "128x64");
-  /*128x32x8*/
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, 8 >(output, options, config, "sgemm_128x32x8_splitk_pi_split8", "128x32");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, 16 >(output, options, config, "sgemm_128x32x8_splitk_pi_split16", "128x32");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, 20 >(output, options, config, "sgemm_128x32x8_splitk_pi_split20", "128x32");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, 24 >(output, options, config, "sgemm_128x32x8_splitk_pi_split24", "128x32");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, 28 >(output, options, config, "sgemm_128x32x8_splitk_pi_split28", "128x32");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, 32 >(output, options, config, "sgemm_128x32x8_splitk_pi_split32", "128x32");
-  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, 64 >(output, options, config, "sgemm_128x32x8_splitk_pi_split64", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 5 >(output, options, config, "sgemm_128x64x8_splitk_pi_split5", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 8 >(output, options, config, "sgemm_128x64x8_splitk_pi_split8", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "sgemm_128x64x8_splitk_pi_split10", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 16 >(output, options, config, "sgemm_128x64x8_splitk_pi_split16", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 20 >(output, options, config, "sgemm_128x64x8_splitk_pi_split20", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 24 >(output, options, config, "sgemm_128x64x8_splitk_pi_split24", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 28 >(output, options, config, "sgemm_128x64x8_splitk_pi_split28", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 32 >(output, options, config, "sgemm_128x64x8_splitk_pi_split32", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 40 >(output, options, config, "sgemm_128x64x8_splitk_pi_split40", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 48 >(output, options, config, "sgemm_128x64x8_splitk_pi_split48", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 56 >(output, options, config, "sgemm_128x64x8_splitk_pi_split56", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 64 >(output, options, config, "sgemm_128x64x8_splitk_pi_split64", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 72 >(output, options, config, "sgemm_128x64x8_splitk_pi_split72", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 80 >(output, options, config, "sgemm_128x64x8_splitk_pi_split80", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 88 >(output, options, config, "sgemm_128x64x8_splitk_pi_split88", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 96 >(output, options, config, "sgemm_128x64x8_splitk_pi_split96", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 104 >(output, options, config, "sgemm_128x64x8_splitk_pi_split104", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 112 >(output, options, config, "sgemm_128x64x8_splitk_pi_split112", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 120 >(output, options, config, "sgemm_128x64x8_splitk_pi_split120", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 128 >(output, options, config, "sgemm_128x64x8_splitk_pi_split128", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 136 >(output, options, config, "sgemm_128x64x8_splitk_pi_split136", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 144 >(output, options, config, "sgemm_128x64x8_splitk_pi_split144", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 152 >(output, options, config, "sgemm_128x64x8_splitk_pi_split152", "128x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 160 >(output, options, config, "sgemm_128x64x8_splitk_pi_split160", "128x64");
+
+  /*128x32x8*/ 
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 5  >(output, options, config, "sgemm_128x32x8_splitk_pi_split5", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 8  >(output, options, config, "sgemm_128x32x8_splitk_pi_split8", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "sgemm_128x32x8_splitk_pi_split10", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 16 >(output, options, config, "sgemm_128x32x8_splitk_pi_split16", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 20 >(output, options, config, "sgemm_128x32x8_splitk_pi_split20", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 24 >(output, options, config, "sgemm_128x32x8_splitk_pi_split24", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 28 >(output, options, config, "sgemm_128x32x8_splitk_pi_split28", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 32 >(output, options, config, "sgemm_128x32x8_splitk_pi_split32", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 40 >(output, options, config, "sgemm_128x32x8_splitk_pi_split40", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 48 >(output, options, config, "sgemm_128x32x8_splitk_pi_split48", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 56 >(output, options, config, "sgemm_128x32x8_splitk_pi_split56", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 64 >(output, options, config, "sgemm_128x32x8_splitk_pi_split64", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 72 >(output, options, config, "sgemm_128x32x8_splitk_pi_split72", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 80 >(output, options, config, "sgemm_128x32x8_splitk_pi_split80", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 88 >(output, options, config, "sgemm_128x32x8_splitk_pi_split88", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 96 >(output, options, config, "sgemm_128x32x8_splitk_pi_split96", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 104 >(output, options, config, "sgemm_128x32x8_splitk_pi_split104", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 112 >(output, options, config, "sgemm_128x32x8_splitk_pi_split112", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 120 >(output, options, config, "sgemm_128x32x8_splitk_pi_split120", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 128 >(output, options, config, "sgemm_128x32x8_splitk_pi_split128", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 136 >(output, options, config, "sgemm_128x32x8_splitk_pi_split136", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 144 >(output, options, config, "sgemm_128x32x8_splitk_pi_split144", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 152 >(output, options, config, "sgemm_128x32x8_splitk_pi_split152", "128x32");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 32, 128>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 160 >(output, options, config, "sgemm_128x32x8_splitk_pi_split160", "128x32");
+  
+  /*64x128*/
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 5 >(output, options, config, "sgemm_64x128x8_splitk_pi_split5", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 8 >(output, options, config, "sgemm_64x128x8_splitk_pi_split8", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "sgemm_64x128x8_splitk_pi_split10", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 16 >(output, options, config, "sgemm_64x128x8_splitk_pi_split16", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 20 >(output, options, config, "sgemm_64x128x8_splitk_pi_split20", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 24 >(output, options, config, "sgemm_64x128x8_splitk_pi_split24", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 28 >(output, options, config, "sgemm_64x128x8_splitk_pi_split28", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 32 >(output, options, config, "sgemm_64x128x8_splitk_pi_split32", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 40 >(output, options, config, "sgemm_64x128x8_splitk_pi_split40", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 48 >(output, options, config, "sgemm_64x128x8_splitk_pi_split48", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 56 >(output, options, config, "sgemm_64x128x8_splitk_pi_split56", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 64 >(output, options, config, "sgemm_64x128x8_splitk_pi_split64", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 72 >(output, options, config, "sgemm_64x128x8_splitk_pi_split72", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 80 >(output, options, config, "sgemm_64x128x8_splitk_pi_split80", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 88 >(output, options, config, "sgemm_64x128x8_splitk_pi_split88", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 96 >(output, options, config, "sgemm_64x128x8_splitk_pi_split96", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 104 >(output, options, config, "sgemm_64x128x8_splitk_pi_split104", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 112 >(output, options, config, "sgemm_64x128x8_splitk_pi_split112", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 120 >(output, options, config, "sgemm_64x128x8_splitk_pi_split120", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 128 >(output, options, config, "sgemm_64x128x8_splitk_pi_split128", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 136 >(output, options, config, "sgemm_64x128x8_splitk_pi_split136", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 144 >(output, options, config, "sgemm_64x128x8_splitk_pi_split144", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 152 >(output, options, config, "sgemm_64x128x8_splitk_pi_split152", "64x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 1>, 160 >(output, options, config, "sgemm_64x128x8_splitk_pi_split160", "64x128");
+  
+  /*32x128*/
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 5 >(output, options, config, "sgemm_32x128x8_splitk_pi_split5", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 8 >(output, options, config, "sgemm_32x128x8_splitk_pi_split8", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "sgemm_32x128x8_splitk_pi_split10", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 16 >(output, options, config, "sgemm_32x128x8_splitk_pi_split16", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 20 >(output, options, config, "sgemm_32x128x8_splitk_pi_split20", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 24 >(output, options, config, "sgemm_32x128x8_splitk_pi_split24", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 28 >(output, options, config, "sgemm_32x128x8_splitk_pi_split28", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 32 >(output, options, config, "sgemm_32x128x8_splitk_pi_split32", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 40 >(output, options, config, "sgemm_32x128x8_splitk_pi_split40", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 48 >(output, options, config, "sgemm_32x128x8_splitk_pi_split48", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 56 >(output, options, config, "sgemm_32x128x8_splitk_pi_split56", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 64 >(output, options, config, "sgemm_32x128x8_splitk_pi_split64", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 72 >(output, options, config, "sgemm_32x128x8_splitk_pi_split72", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 2>, 80 >(output, options, config, "sgemm_32x128x8_splitk_pi_split80", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 88 >(output, options, config, "sgemm_32x128x8_splitk_pi_split88", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 96 >(output, options, config, "sgemm_32x128x8_splitk_pi_split96", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 104 >(output, options, config, "sgemm_32x128x8_splitk_pi_split104", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 112 >(output, options, config, "sgemm_32x128x8_splitk_pi_split112", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 120 >(output, options, config, "sgemm_32x128x8_splitk_pi_split120", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 128 >(output, options, config, "sgemm_32x128x8_splitk_pi_split128", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 136 >(output, options, config, "sgemm_32x128x8_splitk_pi_split136", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 144 >(output, options, config, "sgemm_32x128x8_splitk_pi_split144", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 152 >(output, options, config, "sgemm_32x128x8_splitk_pi_split152", "32x128");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 128, 32>, cutlass::Shape<8, 8, 4>, cutlass::Shape<1, 1, 1>, 160 >(output, options, config, "sgemm_32x128x8_splitk_pi_split160", "32x128");
+
+  /*64x64*/
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 5 >(output, options, config, "sgemm_64x64x8_splitk_pi_split5", "64x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 8 >(output, options, config, "sgemm_64x64x8_splitk_pi_split8", "64x64");
+  results |= profile_sgemm_splitkpi_kernel<cutlass::Shape<8, 64, 64>, cutlass::Shape<8, 8, 8>, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "sgemm_64x64x8_splitk_pi_split10", "64x64");
+
+#endif //#ifdef EXHAUSTIVE_PROF
 
   return results;
 }

tools/test/perf/gemm/volta884_gemm.cu

@@ -0,0 +1,183 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#include "tools/test/perf/gemm/cutlass_volta884_dispatch.h"
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace perf {
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename OutputTile, typename AccumHostType>
+int profile_volta884_gemm_kernel(
+  TestbenchOutput<GemmProblem> &output,
+  TestbenchOptions const &options,
+  Config const &config,
+  std::string const &name,
+  std::string const &algo) {
+
+  int results = 0;
+
+  // compute capability check
+  if (!options.compute_capability(7, 0)) {
+    return 0;
+  }
+
+  typedef typename cutlass::TypeTraits<AccumHostType>::device_type AccumDevType;
+
+#if CUTLASS_ENABLE_TENSOR_CORE_MMA
+  typedef perf::GemmProfiler<
+    cutlass::half_t,
+    cutlass::half_t,
+    AccumHostType,
+    AccumHostType,
+    AccumHostType> GemmProfiler;
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_nn", options, config, algo);
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_nt", options, config, algo);
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_tn", options, config, algo);
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      cutlass::gemm::IdentityBlockSwizzle,
+      true
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_tt", options, config, algo);
+  }
+
+  #endif // if defined(CUTLASS_ENABLE_TENSOR_CORE_MMA)
+
+  return results;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+int profile_volta884_gemm(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
+  int results = 0;
+
+  
+  
+
+  results |= profile_volta884_gemm_kernel<cutlass::Shape<32, 128, 128>, float >(output, options, config, "s884gemm", "128x128");
+
+  results |= profile_volta884_gemm_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t >(output, options, config, "h884gemm", "128x128");
+
+#ifdef EXHAUSTIVE_PROF
+  results |= profile_volta884_gemm_kernel<cutlass::Shape<32, 128, 256>, float >(output, options, config, "s884gemm_256x128", "256x128");
+
+  results |= profile_volta884_gemm_kernel<cutlass::Shape<32, 64, 128>, float >(output, options, config, "s884gemm_128x64", "128x64");
+
+  results |= profile_volta884_gemm_kernel<cutlass::Shape<32, 64, 64>, float >(output, options, config, "s884gemm_64x64", "64x64");
+
+  results |= profile_volta884_gemm_kernel<cutlass::Shape<32, 128, 256>, cutlass::half_t >(output, options, config, "h884gemm_256x128", "256x128");
+
+  results |= profile_volta884_gemm_kernel<cutlass::Shape<32, 64, 128>, cutlass::half_t >(output, options, config, "h884gemm_128x64", "128x64");
+
+  results |= profile_volta884_gemm_kernel<cutlass::Shape<32, 64, 64>, cutlass::half_t >(output, options, config, "h884gemm_64x64", "64x64");
+#endif // defined EXHAUSTIVE_PROF
+
+  return results;
+}
+
+struct Volta884GemmRegistrar {
+  Volta884GemmRegistrar() { RegisterGemmProfileFunc(profile_volta884_gemm); }
+};
+
+volatile Volta884GemmRegistrar _Volta884GemmRegistrar;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace perf
+
+////////////////////////////////////////////////////////////////////////////////////////////////////

tools/test/perf/gemm/volta884_gemm_cta_rasterization_nn.cu

@@ -0,0 +1,242 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#include "tools/test/perf/gemm/cutlass_volta884_dispatch.h"
+
+#ifdef EXHAUSTIVE_PROF
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace perf {
+template <typename OutputTile, typename AccumHostType>
+int profile_volta884_gemm_cta_rasterization_nn_kernel(
+  TestbenchOutput<GemmProblem> &output,
+  TestbenchOptions const &options,
+  Config const &config,
+  std::string const &name,
+  std::string const &algo) {
+
+  int results = 0;
+
+  // compute capability check
+  if (!options.compute_capability(7, 0)) {
+    return 0;
+  }
+
+  typedef typename cutlass::TypeTraits<AccumHostType>::device_type AccumDevType;
+
+#if CUTLASS_ENABLE_TENSOR_CORE_MMA
+  typedef perf::GemmProfiler<
+    cutlass::half_t,
+    cutlass::half_t,
+    AccumHostType,
+    AccumHostType,
+    AccumHostType> GemmProfiler;
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_1_one_nn", options, config, algo + "_row_1_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_1_B_nn", options, config, algo + "_row_1_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_2_one_nn", options, config, algo + "_row_2_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_2_B_nn", options, config, algo + "_row_2_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_1_one_nn", options, config, algo + "_col_1_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_1_B_nn", options, config, algo + "_col_1_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_2_one_nn", options, config, algo + "_col_2_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_2_B_nn", options, config, algo + "_col_2_B");
+  }
+#endif // if defined(CUTLASS_ENABLE_TENSOR_CORE_MMA)
+
+  return results;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+int profile_volta884_gemm_cta_rasterization_nn(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
+  int results = 0;
+
+  results |= profile_volta884_gemm_cta_rasterization_nn_kernel<cutlass::Shape<32, 128, 128>, float >(output, options, config, "s884gemm", "128x128");
+
+  results |= profile_volta884_gemm_cta_rasterization_nn_kernel<cutlass::Shape<32, 128, 256>, float >(output, options, config, "s884gemm_256x128", "256x128");
+
+  return results;
+}
+
+struct Volta884GemmCTARasterizationNNRegistrar {
+  Volta884GemmCTARasterizationNNRegistrar() { RegisterGemmProfileFunc(profile_volta884_gemm_cta_rasterization_nn); }
+};
+
+volatile Volta884GemmCTARasterizationNNRegistrar _Volta884CTARasterizationNNGemmRegistrar;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace perf
+#endif // if defined(EXHAUSTIVE_PROF)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////

tools/test/perf/gemm/volta884_gemm_cta_rasterization_nt.cu

@@ -0,0 +1,247 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#include "tools/test/perf/gemm/cutlass_volta884_dispatch.h"
+
+#ifdef EXHAUSTIVE_PROF
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace perf {
+template <typename OutputTile, typename AccumHostType>
+int profile_volta884_gemm_cta_rasterization_nt_kernel(
+  TestbenchOutput<GemmProblem> &output,
+  TestbenchOptions const &options,
+  Config const &config,
+  std::string const &name,
+  std::string const &algo) {
+
+  int results = 0;
+
+  // compute capability check
+  if (!options.compute_capability(7, 0)) {
+    return 0;
+  }
+
+  typedef typename cutlass::TypeTraits<AccumHostType>::device_type AccumDevType;
+
+#if CUTLASS_ENABLE_TENSOR_CORE_MMA
+  typedef perf::GemmProfiler<
+    cutlass::half_t,
+    cutlass::half_t,
+    AccumHostType,
+    AccumHostType,
+    AccumHostType> GemmProfiler;
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_1_one_nt", options, config, algo + "_row_1_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_1_B_nt", options, config, algo + "_row_1_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_2_one_nt", options, config, algo + "_row_2_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_2_B_nt", options, config, algo + "_row_2_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_1_one_nt", options, config, algo + "_col_1_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_1_B_nt", options, config, algo + "_col_1_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_2_one_nt", options, config, algo + "_col_2_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_2_B_nt", options, config, algo + "_col_2_B");
+  }
+#endif // if defined(CUTLASS_ENABLE_TENSOR_CORE_MMA)
+
+  return results;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+int profile_volta884_gemm_cta_rasterization_nt(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
+  int results = 0;
+
+  results |= profile_volta884_gemm_cta_rasterization_nt_kernel<cutlass::Shape<32, 128, 128>, float >(output, options, config, "s884gemm", "128x128");
+
+  results |= profile_volta884_gemm_cta_rasterization_nt_kernel<cutlass::Shape<32, 128, 256>, float >(output, options, config, "s884gemm_256x128", "256x128");
+
+  results |= profile_volta884_gemm_cta_rasterization_nt_kernel<cutlass::Shape<32, 64, 128>, float >(output, options, config, "s884gemm_128x64", "128x64");
+
+  results |= profile_volta884_gemm_cta_rasterization_nt_kernel<cutlass::Shape<32, 64, 64>, float >(output, options, config, "s884gemm_64x64", "64x64");
+
+  return results;
+}
+
+struct Volta884GemmCTARasterizationNTRegistrar {
+  Volta884GemmCTARasterizationNTRegistrar() { RegisterGemmProfileFunc(profile_volta884_gemm_cta_rasterization_nt); }
+};
+
+volatile Volta884GemmCTARasterizationNTRegistrar _Volta884CTARasterizationNTGemmRegistrar;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace perf
+
+#endif // if defined(EXHAUSTIVE_PROF)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////

tools/test/perf/gemm/volta884_gemm_cta_rasterization_tn.cu

@@ -0,0 +1,247 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#include "tools/test/perf/gemm/cutlass_volta884_dispatch.h"
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace perf {
+template <typename OutputTile, typename AccumHostType>
+int profile_volta884_gemm_cta_rasterization_tn_kernel(
+  TestbenchOutput<GemmProblem> &output,
+  TestbenchOptions const &options,
+  Config const &config,
+  std::string const &name,
+  std::string const &algo) {
+
+  int results = 0;
+
+  // compute capability check
+  if (!options.compute_capability(7, 0)) {
+    return 0;
+  }
+
+  typedef typename cutlass::TypeTraits<AccumHostType>::device_type AccumDevType;
+
+#if CUTLASS_ENABLE_TENSOR_CORE_MMA
+  typedef perf::GemmProfiler<
+    cutlass::half_t,
+    cutlass::half_t,
+    AccumHostType,
+    AccumHostType,
+    AccumHostType> GemmProfiler;
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_1_one_tn", options, config, algo + "_row_1_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_1_B_tn", options, config, algo + "_row_1_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_2_one_tn", options, config, algo + "_row_2_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_2_B_tn", options, config, algo + "_row_2_B");
+  }
+
+#ifdef EXHAUSTIVE_PROF
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_1_one_tn", options, config, algo + "_col_1_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_1_B_tn", options, config, algo + "_col_1_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_2_one_tn", options, config, algo + "_col_2_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_2_B_tn", options, config, algo + "_col_2_B");
+  }
+#endif // if defined(EXHAUSTIVE_PROF)
+#endif // if defined(CUTLASS_ENABLE_TENSOR_CORE_MMA)
+
+  return results;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+int profile_volta884_gemm_cta_rasterization_tn(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
+  int results = 0;
+
+  results |= profile_volta884_gemm_cta_rasterization_tn_kernel<cutlass::Shape<32, 128, 128>, float >(output, options, config, "s884gemm", "128x128");
+
+#ifdef EXHAUSTIVE_PROF
+  results |= profile_volta884_gemm_cta_rasterization_tn_kernel<cutlass::Shape<32, 128, 256>, float >(output, options, config, "s884gemm_256x128", "256x128");
+
+  results |= profile_volta884_gemm_cta_rasterization_tn_kernel<cutlass::Shape<32, 64, 128>, float >(output, options, config, "s884gemm_128x64", "128x64");
+
+  results |= profile_volta884_gemm_cta_rasterization_tn_kernel<cutlass::Shape<32, 64, 64>, float >(output, options, config, "s884gemm_64x64", "64x64");
+#endif // if defined( EXHAUSTIVE_PROF)
+
+  return results;
+}
+
+struct Volta884GemmCTARasterizationTNRegistrar {
+  Volta884GemmCTARasterizationTNRegistrar() { RegisterGemmProfileFunc(profile_volta884_gemm_cta_rasterization_tn); }
+};
+
+volatile Volta884GemmCTARasterizationTNRegistrar _Volta884CTARasterizationTNGemmRegistrar;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace perf
+
+////////////////////////////////////////////////////////////////////////////////////////////////////

tools/test/perf/gemm/volta884_gemm_cta_rasterization_tt.cu

@@ -0,0 +1,255 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#include "tools/test/perf/gemm/cutlass_volta884_dispatch.h"
+
+#ifdef EXHAUSTIVE_PROF
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace perf {
+template <typename OutputTile, typename AccumHostType>
+int profile_volta884_gemm_cta_rasterization_tt_kernel(
+  TestbenchOutput<GemmProblem> &output,
+  TestbenchOptions const &options,
+  Config const &config,
+  std::string const &name,
+  std::string const &algo) {
+
+  int results = 0;
+
+  // compute capability check
+  if (!options.compute_capability(7, 0)) {
+    return 0;
+  }
+
+  typedef typename cutlass::TypeTraits<AccumHostType>::device_type AccumDevType;
+
+#if CUTLASS_ENABLE_TENSOR_CORE_MMA
+  typedef perf::GemmProfiler<
+    cutlass::half_t,
+    cutlass::half_t,
+    AccumHostType,
+    AccumHostType,
+    AccumHostType> GemmProfiler;
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>,
+      true
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_1_one_tt", options, config, algo + "_row_1_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>,
+      true
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_1_B_tt", options, config, algo + "_row_1_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>,
+      true
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_2_one_tt", options, config, algo + "_row_2_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::RowMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>,
+      true
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_row_2_B_tt", options, config, algo + "_row_2_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::OneDirection>,
+      true
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_1_one_tt", options, config, algo + "_col_1_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<1, cutlass::gemm::swizzleDirection::Boustrophedon>,
+      true
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_1_B_tt", options, config, algo + "_col_1_B");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::OneDirection>,
+      true
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_2_one_tt", options, config, algo + "_col_2_one");
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2,
+      cutlass::gemm::LinearScaling<AccumDevType>,
+      typename cutlass::gemm::ColumnMajorBlockSwizzle<2, cutlass::gemm::swizzleDirection::Boustrophedon>,
+      true
+    > GemmTraits;
+
+    typedef Volta884GemmDispatch<GemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_col_2_B_tt", options, config, algo + "_col_2_B");
+  }
+#endif // if defined(CUTLASS_ENABLE_TENSOR_CORE_MMA)
+
+  return results;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+int profile_volta884_gemm_cta_rasterization_tt(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
+  int results = 0;
+
+  results |= profile_volta884_gemm_cta_rasterization_tt_kernel<cutlass::Shape<32, 128, 128>, float >(output, options, config, "s884gemm", "128x128");
+
+  results |= profile_volta884_gemm_cta_rasterization_tt_kernel<cutlass::Shape<32, 128, 256>, float >(output, options, config, "s884gemm_256x128", "256x128");
+
+  results |= profile_volta884_gemm_cta_rasterization_tt_kernel<cutlass::Shape<32, 64, 128>, float >(output, options, config, "s884gemm_128x64", "128x64");
+
+  results |= profile_volta884_gemm_cta_rasterization_tt_kernel<cutlass::Shape<32, 64, 64>, float >(output, options, config, "s884gemm_64x64", "64x64");
+
+  return results;
+}
+
+struct Volta884GemmCTARasterizationTTRegistrar {
+  Volta884GemmCTARasterizationTTRegistrar() { RegisterGemmProfileFunc(profile_volta884_gemm_cta_rasterization_tt); }
+};
+
+volatile Volta884GemmCTARasterizationTTRegistrar _Volta884CTARasterizationTTGemmRegistrar;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace perf
+
+#endif // if defined(EXHAUSTIVE_PROF)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////

tools/test/perf/gemm/volta884_gemm_splitK.cu

@@ -0,0 +1,324 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#include "tools/test/perf/gemm/cutlass_volta884_dispatch_splitK_PI.h"
+#include "cutlass/reduction/batched_reduction_traits.h"
+#include "cutlass/gemm/device_gemm_traits.h"
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace perf {
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename OutputTile, typename AccumHostType, typename threadReductionShape, int splits_count>
+int profile_volta884_gemm_splitkpi_kernel(
+  TestbenchOutput<GemmProblem> &output,
+  TestbenchOptions const &options,
+  Config const &config,
+  std::string const &name,
+  std::string const &algo) {
+
+  int results = 0;
+
+  // compute capability check
+  if (!options.compute_capability(7, 0)) {
+    return 0;
+  }
+
+  typedef typename cutlass::TypeTraits<AccumHostType>::device_type AccumDevType;
+
+#if CUTLASS_ENABLE_TENSOR_CORE_MMA
+  typedef perf::GemmProfiler<
+    cutlass::half_t,
+    cutlass::half_t,
+    cutlass::half_t,
+    AccumHostType,
+    AccumHostType> GemmProfiler;
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2
+    > GemmTraits;
+
+    /*batched reduction traits*/
+    typedef cutlass::reduction::BatchedReductionTraits<AccumDevType,
+      half,
+      half,
+      AccumDevType,
+      AccumDevType,
+      splits_count,
+      cutlass::Shape<1, 1, 128>,
+      cutlass::Shape<1, 1, 64>,
+      threadReductionShape >
+      BatchedReductionTraits;
+
+    // create a device gemm 
+    typedef typename cutlass::gemm::SplitkPIGemmTraits<GemmTraits, BatchedReductionTraits> deviceGemmTraits;
+    typedef Volta884GemmDispatchSplitKPI<deviceGemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_nn", options, config, algo);
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kColumnMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2
+    > GemmTraits;
+
+    /*batched reduction traits*/
+    typedef cutlass::reduction::BatchedReductionTraits<AccumDevType,
+      half,
+      half,
+      AccumDevType,
+      AccumDevType,
+      splits_count,
+      cutlass::Shape<1, 1, 128>,
+      cutlass::Shape<1, 1, 64>,
+      threadReductionShape >
+      BatchedReductionTraits;
+
+    // create a device gemm 
+    typedef typename cutlass::gemm::SplitkPIGemmTraits<GemmTraits, BatchedReductionTraits> deviceGemmTraits;
+    typedef Volta884GemmDispatchSplitKPI<deviceGemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_nt", options, config, algo);
+  }
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kColumnMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2
+    > GemmTraits;
+
+    /*batched reduction traits*/
+    typedef cutlass::reduction::BatchedReductionTraits<AccumDevType,
+      half,
+      half,
+      AccumDevType,
+      AccumDevType,
+      splits_count,
+      cutlass::Shape<1, 1, 128>,
+      cutlass::Shape<1, 1, 64>,
+      threadReductionShape >
+      BatchedReductionTraits;
+
+    // create a device gemm 
+    typedef typename cutlass::gemm::SplitkPIGemmTraits<GemmTraits, BatchedReductionTraits> deviceGemmTraits;
+    typedef Volta884GemmDispatchSplitKPI<deviceGemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_tn", options, config, algo);
+}
+
+  {
+    typedef cutlass::gemm::Volta884GemmTraits<
+      cutlass::MatrixLayout::kRowMajor,
+      cutlass::MatrixLayout::kRowMajor,
+      OutputTile,
+      cutlass::Shape<32, 64, 64>,
+      AccumDevType,
+      AccumDevType,
+      AccumDevType,
+      2
+    > GemmTraits;
+
+    /*batched reduction traits*/
+    typedef cutlass::reduction::BatchedReductionTraits<AccumDevType,
+      half,
+      half,
+      AccumDevType,
+      AccumDevType,
+      splits_count,
+      cutlass::Shape<1, 1, 128>,
+      cutlass::Shape<1, 1, 64>,
+      threadReductionShape >
+      BatchedReductionTraits;
+
+    // create a device gemm 
+    typedef typename cutlass::gemm::SplitkPIGemmTraits<GemmTraits, BatchedReductionTraits> deviceGemmTraits;
+    typedef Volta884GemmDispatchSplitKPI<deviceGemmTraits> Dispatch;
+
+    results |= profile_gemm<Dispatch, GemmProfiler>(output, name + "_tt", options, config, algo);
+  }
+
+  #endif // if defined(CUTLASS_ENABLE_TENSOR_CORE_MMA)
+
+  return results;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+int profile_volta884_gemm_splitkpi(TestbenchOutput<GemmProblem> &output, TestbenchOptions const &options, Config const &config) {
+  int results = 0;
+
+  //results |= profile_volta884_gemm_kernel<cutlass::Shape<32, 128, 128>, float >(output, options, config, "s884gemm", "128x128");
+
+  // half accum
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 5 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits5", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 8 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits8", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits10", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 16 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits16", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 20 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits20", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 24 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits24", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 28 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits28", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 32 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits32", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 40 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits40", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 48 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits48", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 56 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits56", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 64 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits64", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 72 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits72", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 2>, 80 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits80", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 88 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits88", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 96 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits96", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 104 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits104", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 112 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits112", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 120 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits120", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 128 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits128", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 136 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits136", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 144 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits144", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 152 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits152", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, cutlass::half_t, cutlass::Shape<1, 1, 1>, 160 >(output, options, config, "h884gemm_128x128x32_splitk_pi_splits160", "128x128");
+
+  // float accum
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 5 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits5", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 8 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits8", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits10", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 16 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits16", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 20 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits20", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 24 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits24", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 28 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits28", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 32 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits32", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 40 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits40", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 48 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits48", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 56 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits56", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 64 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits64", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 72 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits72", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 2>, 80 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits80", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 88 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits88", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 96 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits96", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 104 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits104", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 112 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits112", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 120 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits120", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 128 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits128", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 136 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits136", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 144 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits144", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 152 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits152", "128x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 128>, float, cutlass::Shape<1, 1, 1>, 160 >(output, options, config, "s884gemm_128x128x32_splitk_pi_splits160", "128x128");
+
+#ifdef EXHAUSTIVE_PROF
+  // float accum 128x64
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 5 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits5", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 8 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits8", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits10", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 16 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits16", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 20 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits20", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 24 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits24", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 28 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits28", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 32 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits32", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 40 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits40", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 48 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits48", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 56 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits56", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 64 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits64", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 72 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits72", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 2>, 80 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits80", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 88 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits88", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 96 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits96", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 104 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits104", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 112 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits112", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 120 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits120", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 128 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits128", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 136 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits136", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 144 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits144", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 152 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits152", "128x64");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 64, 128>, float, cutlass::Shape<1, 1, 1>, 160 >(output, options, config, "s884gemm_128x64x32_splitk_pi_splits160", "128x64");
+
+  // float accum 64x128
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 5 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits5", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 8 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits8", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 10 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits10", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 16 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits16", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 20 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits20", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 24 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits24", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 28 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits28", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 32 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits32", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 40 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits40", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 48 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits48", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 56 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits56", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 64 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits64", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 72 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits72", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 2>, 80 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits80", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 88 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits88", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 96 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits96", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 104 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits104", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 112 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits112", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 120 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits120", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 128 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits128", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 136 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits136", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 144 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits144", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 152 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits152", "64x128");
+  results |= profile_volta884_gemm_splitkpi_kernel<cutlass::Shape<32, 128, 64>, float, cutlass::Shape<1, 1, 1>, 160 >(output, options, config, "s884gemm_64x128x32_splitk_pi_splits160", "64x128");
+#endif //#ifdef EXHAUSTIVE_PROF
+  return results;
+}
+
+struct Volta884GemmSplitKPIRegistrar {
+  Volta884GemmSplitKPIRegistrar() { RegisterGemmProfileFunc(profile_volta884_gemm_splitkpi); }
+};
+
+volatile Volta884GemmSplitKPIRegistrar _Volta884GemmSplitKPIRegistrar;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace perf
+
+////////////////////////////////////////////////////////////////////////////////////////////////////

tools/test/perf/gemm/wmma_binary_gemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -30,7 +30,7 @@
 #include "cutlass/wmma_matrix.h"
 #ifdef CUTLASS_USE_WMMA_API
 #ifdef CUTLASS_USE_SUBBYTE_WMMA
-
+#pragma warning( disable : 4503)
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 #include "cutlass/gemm/gemm.h"

tools/test/perf/gemm/wmma_gemm.cu

@@ -1,27 +1,27 @@
 /***************************************************************************************************
-* Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
-*
-* Redistribution and use in source and binary forms, with or without modification, are permitted
-* provided that the following conditions are met:
-*     * Redistributions of source code must retain the above copyright notice, this list of
-*       conditions and the following disclaimer.
-*     * Redistributions in binary form must reproduce the above copyright notice, this list of
-*       conditions and the following disclaimer in the documentation and/or other materials
-*       provided with the distribution.
-*     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
-*       to endorse or promote products derived from this software without specific prior written
-*       permission.
-*
-* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
-* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
-* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
-* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
-* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*
-**************************************************************************************************/
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
 
 #include "cutlass/wmma_matrix.h"
 #ifdef CUTLASS_USE_WMMA_API
@@ -49,7 +49,11 @@ struct WmmaGemmDispatch {
   /// Indicate warp-level GEMM
   static bool const kThreadMultiplyAdd = false;
 
+  #if CUTLASS_ENABLE_CUBLAS
   static bool const kRunCuBLAS = true;
+  #else
+  static bool const kRunCuBLAS = false;
+  #endif
 
   static cutlass::MatrixLayout::Kind const kLayoutA = Traits::kLayoutA;
   static cutlass::MatrixLayout::Kind const kLayoutB = Traits::kLayoutB;
@@ -138,8 +142,8 @@ int profile_wmma_gemm_f32(TestbenchOutput<GemmProblem> &output, TestbenchOptions
 
   {
     typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
-      cutlass::MatrixLayout::kRowMajor>
-      WmmaGemmTraits;
+                                        cutlass::MatrixLayout::kRowMajor>
+    WmmaGemmTraits;
 
     typedef WmmaGemmDispatch<WmmaGemmTraits> Dispatch;
 
@@ -148,8 +152,8 @@ int profile_wmma_gemm_f32(TestbenchOutput<GemmProblem> &output, TestbenchOptions
 
   {
     typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
-      cutlass::MatrixLayout::kColumnMajor>
-      WmmaGemmTraits;
+                                        cutlass::MatrixLayout::kColumnMajor>
+    WmmaGemmTraits;
 
     typedef WmmaGemmDispatch<WmmaGemmTraits> Dispatch;
 
@@ -158,7 +162,7 @@ int profile_wmma_gemm_f32(TestbenchOutput<GemmProblem> &output, TestbenchOptions
 
   {
     typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
-      cutlass::MatrixLayout::kColumnMajor>
+                                        cutlass::MatrixLayout::kColumnMajor>
       WmmaGemmTraits;
 
     typedef WmmaGemmDispatch<WmmaGemmTraits> Dispatch;
@@ -168,7 +172,7 @@ int profile_wmma_gemm_f32(TestbenchOutput<GemmProblem> &output, TestbenchOptions
 
   {
     typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
-      cutlass::MatrixLayout::kRowMajor>
+                                        cutlass::MatrixLayout::kRowMajor>
       WmmaGemmTraits;
 
     typedef WmmaGemmDispatch<WmmaGemmTraits> Dispatch;
@@ -183,9 +187,9 @@ int profile_wmma_gemm_f32(TestbenchOutput<GemmProblem> &output, TestbenchOptions
 
 template <typename DummyT>
 int profile_wmma_gemm_f16(
-  TestbenchOutput<GemmProblem> &output,
-  TestbenchOptions const &options,
-  Config const &config) {
+    TestbenchOutput<GemmProblem> &output,
+    TestbenchOptions const &options,
+    Config const &config) {
 
   typedef perf::GemmProfiler<
     cutlass::half_t,
@@ -278,9 +282,9 @@ int profile_wmma_gemm_f16(
 
 template <typename DummyT>
 int profile_wmma_4_gemm_f16(
-  TestbenchOutput<GemmProblem> &output,
-  TestbenchOptions const &options,
-  Config const &config) {
+    TestbenchOutput<GemmProblem> &output,
+    TestbenchOptions const &options,
+    Config const &config) {
 
   typedef perf::GemmProfiler<
     cutlass::half_t,
@@ -547,11 +551,11 @@ struct WmmaGemmRegistrar {
     RegisterGemmProfileFunc(profile_wmma_gemm_f32<void>);
     RegisterGemmProfileFunc(profile_wmma_gemm_f16<void>);
 
-    //#ifdef EXHAUSTIVE_PROF
+//#ifdef EXHAUSTIVE_PROF
     RegisterGemmProfileFunc(profile_wmma_4_gemm_f16<void>);
     //fp32 accum with fp16 input and output
     RegisterGemmProfileFunc(profile_wmma_4_fp16_sgemm_fp16<void>);
-    //#endif  // defined EXHAUSTIVE_PROF
+//#endif  // defined EXHAUSTIVE_PROF
   }
 };
 
@@ -564,4 +568,3 @@ volatile WmmaGemmRegistrar _WmmaGemmRegistrar;
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 #endif  // defined CUTLASS_USE_WMMA_API
-

tools/test/perf/gemm/wmma_integer_gemm.cu

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -29,8 +29,8 @@
 
 #include "cutlass/wmma_matrix.h"
 #ifdef CUTLASS_USE_WMMA_API
-#ifdef CUTLASS_USE_SUBBYTE_WMMA
-
+#ifdef CUTLASS_USE_INT_WMMA
+#pragma warning( disable : 4503)
 #include "cutlass/gemm/gemm.h"
 #include "cutlass/gemm/wmma_gemm_traits.h"
 #include "tools/test/perf/gemm/cutlass_dispatch.h"
@@ -92,6 +92,7 @@ struct WmmaIntegerGemmDispatch {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
+#ifdef CUTLASS_USE_SUBBYTE_WMMA
 template<typename Traits>
 struct WmmaIntegerGemmDispatch<Traits,
                                cutlass::Vector<cutlass::int4_t, 8>,
@@ -209,6 +210,7 @@ struct WmmaIntegerGemmDispatch<Traits,
   /// Launches kernel
   cudaError_t operator()() { return Gemm::launch(params); }
 };
+#endif //ifdef CUTLASS_USE_SUBBYTE_WMMA
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
@@ -221,7 +223,7 @@ int profile_wmma_integer_gemm(TestbenchOutput<GemmProblem> &output, TestbenchOpt
   int results = 0;
 
   // compute capability check
-  if (!options.compute_capability(7, 5)) {
+  if (!options.compute_capability(7, 2)) {
     return 0;
   }
 
@@ -398,6 +400,7 @@ int profile_wmma_integer_gemm(TestbenchOutput<GemmProblem> &output, TestbenchOpt
     return 0;
   }
 
+#ifdef CUTLASS_USE_SUBBYTE_WMMA
   {
     typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
                                           cutlass::MatrixLayout::kColumnMajor,
@@ -451,6 +454,7 @@ int profile_wmma_integer_gemm(TestbenchOutput<GemmProblem> &output, TestbenchOpt
 
     results |= profile_gemm<Dispatch, GemmProfiler>(output, "wmma_integer_gemm_u4_tn", options, config);
   }
+#endif //ifdef CUTLASS_USE_SUBBYTE_WMMA
 
   return results;
 }
@@ -461,7 +465,7 @@ int profile_wmma_integer_gemm(TestbenchOutput<GemmProblem> &output, TestbenchOpt
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-#else // ! CUTLASS_USE_SUBBYTE_WMMA
+#else // ! CUTLASS_USE_INT_WMMA
 
 namespace perf {
 

tools/test/perf/performance_result.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -28,7 +28,6 @@
 #include "cutlass/matrix_traits.h"
 #include "tools/util/command_line.h"
 #include "tools/test/perf/provider.h"
-
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 namespace perf {
@@ -175,7 +174,6 @@ inline std::ostream &operator<<(std::ostream &out, GemmProblem const &problem) {
 
   return out;
 }
-
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 /// Result object

tools/test/perf/provider.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:

tools/test/perf/testbench_configs.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -45,9 +45,9 @@ struct Config {
   // kernel to run
   std::vector<std::string> kernels;
 
-  /// Range of problem sizes
-  GemmProblemRange problem_range;
-
+  /// Range of problem sizes for GEMM
+  GemmProblemRange gemm_problem_range;
+  
   // Reference GFLOPs
   double gflops_ref;
 
@@ -121,8 +121,12 @@ struct TestbenchConfigs {
           if (item.compare("Kernel") == 0) kernel_idx = idx;
           if (item.compare("Beta") == 0) beta_idx = idx;
           if (item.compare("M") == 0) m_idx = idx;
-          if (item.compare("N") == 0) n_idx = idx;
-          if (item.compare("K") == 0) k_idx = idx;
+          if (item.compare("N") == 0) { 
+            n_idx = idx;
+          }
+          if (item.compare("K") == 0) { 
+            k_idx = idx; 
+          }
           if (item.compare("GFLOPs") == 0) gflops_idx = idx;
           if (item.compare("Runtime") == 0) runtime_idx = idx;
           if (item.compare("SOL") == 0) peak_throughput_idx = idx;
@@ -150,9 +154,9 @@ struct TestbenchConfigs {
           config.alpha = options.alpha;
           config.beta = strtod(tokens[beta_idx].c_str(), NULL);
           config.kernels.push_back(tokens[kernel_idx]);
-          config.problem_range.M = Range((int)strtol(tokens[m_idx].c_str(), NULL, 10));
-          config.problem_range.N = Range((int)strtol(tokens[n_idx].c_str(), NULL, 10));
-          config.problem_range.K = Range((int)strtol(tokens[k_idx].c_str(), NULL, 10));
+          config.gemm_problem_range.M = Range(tokens[m_idx]);
+          config.gemm_problem_range.N = Range(tokens[n_idx]);
+          config.gemm_problem_range.K = Range(tokens[k_idx]);
           config.gflops_ref = strtod(tokens[gflops_idx].c_str(), NULL);
           config.runtime_ref = strtod(tokens[runtime_idx].c_str(), NULL);
           config.peak_throughput_ref = strtod(tokens[peak_throughput_idx].c_str(), NULL);
@@ -172,7 +176,7 @@ struct TestbenchConfigs {
     for (int i = 0; i < options.kernels.size(); i++) {
       config.kernels.push_back(options.kernels[i]);
     }
-    config.problem_range = options.problem_range;
+    config.gemm_problem_range = options.gemm_problem_range;
     configs.push_back(config);
   }
 

tools/test/perf/testbench_options.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met:
@@ -66,6 +66,11 @@ struct Range {
   Range(int _start, int _end, int _increment = 1, Operator _op = Add)
       : start(_start), end(_end), increment(_increment), increment_op(_op) {}
 
+  Range(std::string _start) : increment(1), increment_op(Add) {
+    start = end = (int)strtol(_start.c_str(), NULL, 10);
+  }
+
+
   /// Returns the next item in series
   int next(int val) const {
     switch (increment_op) {
@@ -162,8 +167,6 @@ struct GemmProblemRange {
   }
 };
 
-////////////////////////////////////////////////////////////////////////////////////////////////////
-
 /// Defines a vector of string pairs
 typedef std::vector<std::pair<std::string, std::string> > KeyValueVector;
 
@@ -391,8 +394,8 @@ struct TestbenchOptions {
   /// Scalar value for GEMM
   double beta;
 
-  /// Range of problem sizes
-  GemmProblemRange problem_range;
+  /// Range of GEMM problem sizes
+  GemmProblemRange gemm_problem_range;
 
   /// If true, kernels are not executed, and no sleep waits are inserted
   bool dry_run;
@@ -418,7 +421,7 @@ struct TestbenchOptions {
       : initial_distribution(args),
         execution_mode(ExecutionMode::Profile),
         save_workspace_mode(WorkspaceSaveMode::Never),
-        problem_range(args),
+        gemm_problem_range(args),
         dry_run(false),
         sleep_time(1) {
 
@@ -473,6 +476,8 @@ struct TestbenchOptions {
         "igemm",
         "wmma_gemm",
         "wmma_gemm_f16",
+        "s884gemm",
+        "h884gemm",
         "wmma_binary_gemm",
         "wmma_integer_gemm",
         0
@@ -480,7 +485,8 @@ struct TestbenchOptions {
       char const *layouts[] = {"nn", "nt", "tn", "tt", 0};
       for (int i = 0; gemms[i]; ++i) {
         for (int j = 0; layouts[j]; ++j) {
-          if ((std::string(gemms[i]).compare("wmma_binary_gemm") == 0 ||
+          if ((
+               std::string(gemms[i]).compare("wmma_binary_gemm") == 0 ||
                std::string(gemms[i]).compare("wmma_integer_gemm") == 0)
                && std::string(layouts[j]).compare("tn") != 0) {
             continue;
@@ -488,7 +494,7 @@ struct TestbenchOptions {
           kernels.push_back(std::string(gemms[i]) + "_" + layouts[j]);
         }
       }
-
+      
     }
   }
 
@@ -596,15 +602,14 @@ struct TestbenchOptions {
         << "    Height of GEMM problem (number of rows of C). May specify a range with optional "
            "step size.\n"
 
-        << "  --n=<width>[:max width[:step]]                "
+        << "  --n=<width>[:max width[:step]] (GEMM-specific)"
         << "    Width of GEMM problem (number of columns of C). May specify a range with optional "
            "step size.\n"
 
-        << "  --k=<depth>[:max depth[:step]]                "
+        << "  --k=<depth>[:max depth[:step]] (GEMM-specific)"
         << "    Size of inner dimension of A and B. May specify a range with optional step size.\n"
-
         << "  --batch=<batch>                               "
-        << "    Number of batches for a bached gemm. "
+        << "    Number of batches for a batched gemm.\n"
 
         << "  --kernels=<{s|d|h|i|wmma_|wmma_binary_|wmma_integer_}gemm_{nn,nt,tn,tt}>\n"
         << "                                                "
@@ -641,13 +646,14 @@ struct TestbenchOptions {
     out << "\n\n"
         << "Example usage:\n\n"
 
-        << "# Runs one problem size for all kernels\n"
+        << "# Runs one problem size for all GEMM kernels\n"
         << "./tools/test/perf/cutlass_perf_test --m=10240 --n=1024 --k=1024\n\n"
 
         << "# Varies GEMM K dimension for SGEMM and IGEMM with column-major multiplicands\n"
         << "./tools/test/perf/cutlass_perf_test --m=10240 --n=4096 --k=1024:8192:128 "
            "--kernels=sgemm_nn,igemm_nn\n\n"
-
+        << " # Executes GEMM kernel on Volta Tensor Cores\n"
+        << " $ ./tools/test/perf/cutlass_perf_test --kernels=s884gemm_nt\n\n"
         << std::flush;
   }
 };

tools/test/perf/testbench_output.h

@@ -1,5 +1,5 @@
 /***************************************************************************************************
- * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without modification, are permitted
  * provided that the following conditions are met: