CUTLASS 2.1 (#83)

CUTLASS 2.1 contributes: - BLAS-style host-side API added to CUTLASS Library - Planar Complex GEMM kernels targeting Volta and Turing Tensor Cores - Minor enhancements and bug fixes
2020-04-07 13:51:25 -07:00
parent 7c0cd26d13
commit 96dab34ad9
196 changed files with 20653 additions and 1995 deletions
@@ -23,4 +23,5 @@
 cutlass_test_unit_add_executable(
  cutlass_test_unit_epilogue_thread
  linear_combination.cu
-  )
+  linear_combination_planar_complex.cu
+)
@@ -0,0 +1,280 @@
+/***************************************************************************************************
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted
+ * provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice, this list of
+ *       conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+ *       to endorse or promote products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ **************************************************************************************************/
+/*! \file
+    \brief Unit tests for thread-level GEMM
+*/
+
+#include "../../common/cutlass_unit_test.h"
+
+#include "cutlass/epilogue/thread/linear_combination_planar_complex.h"
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace test {
+namespace epilogue {
+namespace thread {
+
+using FunctorPlanarComplexF32F32 = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
+  float, 
+  4,
+  float, 
+  float>;
+
+__global__ void epilogue_thread_functor_planar_complex_f32_f32(
+  float *output_ptr, 
+  float const *accum_ptr, 
+  float const *source_ptr,
+  typename FunctorPlanarComplexF32F32::Params params) {
+
+  FunctorPlanarComplexF32F32 linear_combination_op(params);
+
+  auto accum = *reinterpret_cast<cutlass::ArrayPlanarComplex<float , 4> const *>(accum_ptr);
+  auto source = *reinterpret_cast<cutlass::ArrayPlanarComplex<float, 4> const *>(source_ptr);
+
+  *reinterpret_cast<cutlass::ArrayPlanarComplex<float, 4>*>(output_ptr) = linear_combination_op(accum, source);
+}
+
+}
+}
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Epilogue_thread_linear_combination_planar_complex, f32) {
+
+  using Element = float;
+  using ElementOutput = float;
+  int const kCount = 4;
+
+  using Functor = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
+    ElementOutput, 
+    kCount,
+    Element, 
+    Element>;
+
+  cutlass::complex<Element> alpha(Element(2), Element(1));
+  cutlass::complex<Element> beta(Element(1), Element(-1));
+
+  typename Functor::Params params(alpha, beta);
+
+  Functor linear_combination_op(params);
+
+  cutlass::ArrayPlanarComplex<ElementOutput, kCount> source;
+  cutlass::ArrayPlanarComplex<Element, kCount> accum;
+
+  // Define arbitrary inputs
+  for (int i = 0; i < kCount; ++i) {
+    accum.real[i] = Element(i * 2);
+    accum.imag[i] = Element((i * 3 % 6) - 3);
+    source.real[i] = ElementOutput((i * 7 % 9) - 4);
+    source.imag[i] = ElementOutput(((i * 5 + 2) % 9) - 4);
+  }
+
+  cutlass::ArrayPlanarComplex<ElementOutput, kCount> destination = linear_combination_op(accum, source);
+
+  // Verify each result
+  for (int i = 0; i < kCount; ++i) {
+    
+    cutlass::complex<Element> expected = alpha * cutlass::complex<Element>(accum.real[i], accum.imag[i]) + 
+      beta * cutlass::complex<Element>(Element(source.real[i]), Element(source.imag[i]));
+
+    cutlass::complex<ElementOutput> got(destination.real[i], destination.imag[i]);
+    
+    EXPECT_TRUE(ElementOutput(expected.real()) == got.real());
+    EXPECT_TRUE(ElementOutput(expected.imag()) == got.imag());
+    EXPECT_TRUE(expected.real() != Element(0) || expected.imag() != Element(0));
+  }
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace test {
+namespace epilogue {
+namespace thread {
+
+using FunctorPlanarComplexF16F32 = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
+  cutlass::half_t, 
+  4,
+  float, 
+  float>;
+
+__global__ void epilogue_thread_functor_planar_complex_f16_f32(
+  cutlass::half_t *output_ptr, 
+  float const *accum_ptr, 
+  cutlass::half_t const *source_ptr,
+  typename FunctorPlanarComplexF16F32::Params params,
+  int N) {
+
+  FunctorPlanarComplexF16F32 linear_combination_op(params);
+
+  
+  auto accum = *reinterpret_cast<cutlass::ArrayPlanarComplex<float , 4> const *>(accum_ptr);   
+  auto source = *reinterpret_cast<cutlass::ArrayPlanarComplex<cutlass::half_t , 4> const *>(source_ptr);
+
+  #pragma unroll 1
+  for (int n = 0; n < N; ++n) {
+    source = linear_combination_op(accum, source);
+  }
+
+  *reinterpret_cast<cutlass::ArrayPlanarComplex<cutlass::half_t , 4>*>(output_ptr) = source;
+}
+
+}
+}
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Epilogue_thread_linear_combination_planar_complex, f16_f32) {
+
+  using Element = float;
+  using ElementOutput = cutlass::half_t;
+  int const kCount = 4;
+
+  using Functor = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
+    ElementOutput, 
+    kCount,
+    Element, 
+    Element>;
+
+  cutlass::complex<Element> alpha(Element(2), Element(1));
+  cutlass::complex<Element> beta(Element(1), Element(-1));
+
+  typename Functor::Params params(alpha, beta);
+
+  Functor linear_combination_op(params);
+
+  cutlass::ArrayPlanarComplex<ElementOutput, kCount> source;
+  cutlass::ArrayPlanarComplex<Element, kCount> accum;
+
+  // Define arbitrary inputs
+  for (int i = 0; i < kCount; ++i) {
+    accum.real[i] = Element(i * 2);
+    accum.imag[i] = Element((i * 3 % 6) - 3);
+    source.real[i] = ElementOutput((i * 7 % 9) - 4);
+    source.imag[i] = ElementOutput(((i * 5 + 2) % 9) - 4);
+  }
+
+  cutlass::ArrayPlanarComplex<ElementOutput, kCount> destination = linear_combination_op(accum, source);
+
+  // Verify each result
+  for (int i = 0; i < kCount; ++i) {
+    
+    cutlass::complex<Element> expected = alpha * cutlass::complex<Element>(accum.real[i], accum.imag[i]) + 
+      beta * cutlass::complex<Element>(Element(source.real[i]), Element(source.imag[i]));
+
+    cutlass::complex<ElementOutput> got(destination.real[i], destination.imag[i]);
+    
+    EXPECT_TRUE(ElementOutput(expected.real()) == got.real());
+    EXPECT_TRUE(ElementOutput(expected.imag()) == got.imag());
+    EXPECT_TRUE(expected.real() != Element(0) || expected.imag() != Element(0));
+  }
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace test {
+namespace epilogue {
+namespace thread {
+
+using FunctorPlanarComplexF16F16 = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
+  cutlass::half_t, 
+  4,
+  cutlass::half_t, 
+  cutlass::half_t>;
+
+__global__ void epilogue_thread_functor_planar_complex_f16_f16(
+  cutlass::half_t *output_ptr, 
+  cutlass::half_t const *accum_ptr, 
+  cutlass::half_t const *source_ptr,
+  typename FunctorPlanarComplexF16F16::Params params,
+  int N) {
+
+  FunctorPlanarComplexF16F16 linear_combination_op(params);
+
+  auto accum = *reinterpret_cast<cutlass::ArrayPlanarComplex<cutlass::half_t , 4> const *>(accum_ptr);
+  auto source = *reinterpret_cast<cutlass::ArrayPlanarComplex<cutlass::half_t , 4> const *>(source_ptr);
+
+  #pragma unroll 1
+  for (int n = 0; n < N; ++n) {
+    source = linear_combination_op(accum, source);
+  }
+
+  *reinterpret_cast<cutlass::ArrayPlanarComplex<cutlass::half_t , 4>*>(output_ptr) = source;
+}
+
+}
+}
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEST(Epilogue_thread_linear_combination_planar_complex, f16_f16) {
+
+  using Element = cutlass::half_t;
+  using ElementOutput = cutlass::half_t;
+  int const kCount = 8;
+
+  using Functor = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
+    ElementOutput, 
+    kCount,
+    Element, 
+    Element>;
+
+  cutlass::complex<Element> alpha(Element(2), Element(1));
+  cutlass::complex<Element> beta(Element(1), Element(-1));
+
+  typename Functor::Params params(alpha, beta);
+
+  Functor linear_combination_op(params);
+
+  cutlass::ArrayPlanarComplex<ElementOutput, kCount> source;
+  cutlass::ArrayPlanarComplex<Element, kCount> accum;
+
+  // Define arbitrary inputs
+  for (int i = 0; i < kCount; ++i) {
+    accum.real[i] = Element(i * 2);
+    accum.imag[i] = Element((i * 3 % 6) - 3);
+    source.real[i] = ElementOutput((i * 7 % 9) - 4);
+    source.imag[i] = ElementOutput(((i * 5 + 2) % 9) - 4);
+  }
+
+  cutlass::ArrayPlanarComplex<ElementOutput, kCount> destination = linear_combination_op(accum, source);
+
+  // Verify each result
+  for (int i = 0; i < kCount; ++i) {
+    
+    cutlass::complex<Element> expected = alpha * cutlass::complex<Element>(accum.real[i], accum.imag[i]) + 
+      beta * cutlass::complex<Element>(Element(source.real[i]), Element(source.imag[i]));
+
+    cutlass::complex<ElementOutput> got(destination.real[i], destination.imag[i]);
+    
+    EXPECT_TRUE(ElementOutput(expected.real()) == got.real());
+    EXPECT_TRUE(ElementOutput(expected.imag()) == got.imag());
+    EXPECT_TRUE(expected.real() != Element(0) || expected.imag() != Element(0));
+  }
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////