CUTLASS 3.0 Hopper GEMMs are GETTs in disguise (#897)
This commit is contained in:
Vijay Thakkar
@@ -0,0 +1,369 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief GETT command line parser to gather semantic modes, their stride order, and extents.
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
#include <iomanip>
|
||||
#include <utility>
|
||||
#include <type_traits>
|
||||
#include <vector>
|
||||
#include <map>
|
||||
#include <algorithm>
|
||||
#include <numeric>
|
||||
|
||||
#include "cutlass/util/command_line.h"
|
||||
|
||||
namespace cutlass {
|
||||
|
||||
// Output shortcuts
|
||||
std::ostream& operator<<(std::ostream& os, std::vector<char> data) {
|
||||
for (auto& a : data) os << a;
|
||||
return os;
|
||||
}
|
||||
|
||||
template <class T>
|
||||
std::ostream& operator<<(std::ostream& os, std::vector<T> data) {
|
||||
for (auto& a : data) os << a << " ";
|
||||
return os;
|
||||
}
|
||||
|
||||
struct GettCommandLine {
|
||||
struct GettProblem {
|
||||
using extent_type = int;
|
||||
using stride_type = int64_t;
|
||||
|
||||
// Row modes: appear in A and C/D
|
||||
std::vector<extent_type> M;
|
||||
std::vector<stride_type> ldAm;
|
||||
std::vector<stride_type> ldCm;
|
||||
|
||||
// Column modes: appear in B and C/D
|
||||
std::vector<extent_type> N;
|
||||
std::vector<stride_type> ldBn;
|
||||
std::vector<stride_type> ldCn;
|
||||
|
||||
// Reduction modes: appear in A and B
|
||||
std::vector<extent_type> K;
|
||||
std::vector<stride_type> ldAk;
|
||||
std::vector<stride_type> ldBk;
|
||||
|
||||
// Batch modes: appear in all in/out tensors
|
||||
std::vector<extent_type> L;
|
||||
std::vector<stride_type> ldAl;
|
||||
std::vector<stride_type> ldBl;
|
||||
std::vector<stride_type> ldCl;
|
||||
};
|
||||
|
||||
static GettProblem
|
||||
parse(int argc, char const* argv[], bool parse_verbose = false) {
|
||||
using extent_type = typename GettProblem::extent_type;
|
||||
using stride_type = typename GettProblem::stride_type;
|
||||
|
||||
cutlass::CommandLine cmd(argc, argv);
|
||||
|
||||
// modeA
|
||||
std::vector<char> a_mode;
|
||||
cmd.get_cmd_line_arguments("modeA", a_mode);
|
||||
|
||||
// modeB
|
||||
std::vector<char> b_mode;
|
||||
cmd.get_cmd_line_arguments("modeB", b_mode);
|
||||
|
||||
// modeC
|
||||
std::vector<char> c_mode;
|
||||
cmd.get_cmd_line_arguments("modeC", c_mode);
|
||||
|
||||
|
||||
// mode_sizes
|
||||
std::map<char,extent_type> mode_size;
|
||||
// First, initialize all modes in a, b, c to make sure they're in map
|
||||
for (char a : a_mode) mode_size[a] = 1;
|
||||
for (char b : b_mode) mode_size[b] = 1;
|
||||
for (char c : c_mode) mode_size[c] = 1;
|
||||
|
||||
// Then, overwrite the ones in -extent
|
||||
std::vector<std::pair<std::string, std::string> > extent_tokens;
|
||||
cmd.get_cmd_line_argument_pairs("extents", extent_tokens);
|
||||
for (auto e : extent_tokens) {
|
||||
if (std::get<0>(e).size() > 1) {
|
||||
std::cerr << "ERROR: Mode name must only be 1 character long.\n";
|
||||
print_usage();
|
||||
exit(1);
|
||||
}
|
||||
char label = std::get<0>(e)[0];
|
||||
int size = std::stoi(std::get<1>(e));
|
||||
mode_size[label] = size;
|
||||
}
|
||||
|
||||
// Print out symbolic modes and their extents
|
||||
if (parse_verbose) {
|
||||
std::cout << "C_" << c_mode << " = A_" << a_mode << " * B_" << b_mode << "\n";
|
||||
for (auto e : mode_size) std::cout << " " << std::get<0>(e) << " : " << std::get<1>(e) << "\n";
|
||||
}
|
||||
|
||||
//
|
||||
// Collect/Compute strides
|
||||
//
|
||||
|
||||
std::map<char,stride_type> mode_ldA;
|
||||
std::map<char,stride_type> mode_ldB;
|
||||
std::map<char,stride_type> mode_ldC;
|
||||
|
||||
{
|
||||
stride_type current;
|
||||
|
||||
current = 1;
|
||||
for (char a : a_mode) { mode_ldA[a] = current; current *= mode_size[a]; }
|
||||
|
||||
current = 1;
|
||||
for (char b : b_mode) { mode_ldB[b] = current; current *= mode_size[b]; }
|
||||
|
||||
current = 1;
|
||||
for (char c : c_mode) { mode_ldC[c] = current; current *= mode_size[c]; }
|
||||
}
|
||||
|
||||
//
|
||||
// Collect mode categories
|
||||
//
|
||||
|
||||
std::vector<char> row_mode; // rows
|
||||
std::vector<char> col_mode; // columns
|
||||
std::vector<char> red_mode; // reductions
|
||||
std::vector<char> bat_mode; // batches
|
||||
|
||||
{
|
||||
std::vector<char> a_label = a_mode;
|
||||
std::vector<char> b_label = b_mode;
|
||||
std::vector<char> c_label = c_mode;
|
||||
|
||||
std::sort(std::begin(a_label), std::end(a_label));
|
||||
std::sort(std::begin(b_label), std::end(b_label));
|
||||
std::sort(std::begin(c_label), std::end(c_label));
|
||||
|
||||
// std::set_intersections to find semantic category of each symbolic mode
|
||||
std::set_intersection(std::begin(a_label), std::end(a_label),
|
||||
std::begin(c_label), std::end(c_label),
|
||||
std::back_inserter(row_mode));
|
||||
|
||||
std::set_intersection(std::begin(b_label), std::end(b_label),
|
||||
std::begin(c_label), std::end(c_label),
|
||||
std::back_inserter(col_mode));
|
||||
|
||||
std::set_intersection(std::begin(a_label), std::end(a_label),
|
||||
std::begin(b_label), std::end(b_label),
|
||||
std::back_inserter(red_mode));
|
||||
|
||||
std::set_intersection(std::begin(row_mode), std::end(row_mode),
|
||||
std::begin(col_mode), std::end(col_mode),
|
||||
std::back_inserter(bat_mode));
|
||||
|
||||
// std::set_difference to remove batch modes from other semantic modes
|
||||
for (char l : bat_mode) {
|
||||
row_mode.erase(std::remove(std::begin(row_mode), std::end(row_mode), l), std::end(row_mode));
|
||||
col_mode.erase(std::remove(std::begin(col_mode), std::end(col_mode), l), std::end(col_mode));
|
||||
red_mode.erase(std::remove(std::begin(red_mode), std::end(red_mode), l), std::end(red_mode));
|
||||
}
|
||||
}
|
||||
|
||||
// Print out the semantic association of each symbolic mode
|
||||
if (parse_verbose) {
|
||||
std::cout << " rows : " << row_mode << '\n';
|
||||
std::cout << " cols : " << col_mode << '\n';
|
||||
std::cout << " reds : " << red_mode << '\n';
|
||||
std::cout << " bats : " << bat_mode << '\n';
|
||||
}
|
||||
|
||||
//
|
||||
// Permute modes
|
||||
//
|
||||
|
||||
// Permute the batched modes to promote coalescing
|
||||
// Sort the batched modes by min(ldAl,ldBl) and tie-broken by the size
|
||||
std::sort(std::begin(bat_mode), std::end(bat_mode), [&](char l1, char l2) {
|
||||
return std::tie(std::min(mode_ldA[l1],mode_ldB[l1]),mode_size[l1])
|
||||
< std::tie(std::min(mode_ldA[l2],mode_ldB[l2]),mode_size[l2]);
|
||||
});
|
||||
// Compute sizes and strides of ordered reduction modes
|
||||
std::vector<extent_type> L;
|
||||
std::vector<stride_type> ldAl;
|
||||
std::vector<stride_type> ldBl;
|
||||
std::vector<stride_type> ldCl;
|
||||
for (char l : bat_mode) {
|
||||
L.push_back(mode_size[l]);
|
||||
ldAl.push_back(mode_ldA[l]);
|
||||
ldBl.push_back(mode_ldB[l]);
|
||||
ldCl.push_back(mode_ldC[l]);
|
||||
}
|
||||
|
||||
// Permute the reduction modes to promote coalescing
|
||||
// Sort the reduction modes by min(ldAk,ldBk) and tie-broken by the size
|
||||
std::sort(std::begin(red_mode), std::end(red_mode), [&](char k1, char k2) {
|
||||
return std::tie(std::min(mode_ldA[k1],mode_ldB[k1]),mode_size[k1])
|
||||
< std::tie(std::min(mode_ldA[k2],mode_ldB[k2]),mode_size[k2]);
|
||||
});
|
||||
// Compute sizes and strides of ordered reduction modes
|
||||
std::vector<extent_type> K;
|
||||
std::vector<stride_type> ldAk;
|
||||
std::vector<stride_type> ldBk;
|
||||
for (char k : red_mode) {
|
||||
K.push_back(mode_size[k]);
|
||||
ldAk.push_back(mode_ldA[k]);
|
||||
ldBk.push_back(mode_ldB[k]);
|
||||
}
|
||||
|
||||
// Permute the row modes to promote coalescing
|
||||
// Sort the row modes by min(ldAm,ldCm) and tie-broken by ldAm
|
||||
std::sort(std::begin(row_mode), std::end(row_mode), [&](char m1, char m2) {
|
||||
return std::tie(std::min(mode_ldA[m1],mode_ldC[m1]),mode_ldA[m1])
|
||||
< std::tie(std::min(mode_ldA[m2],mode_ldC[m2]),mode_ldA[m2]);
|
||||
});
|
||||
// Compute sizes and strides of ordered row modes
|
||||
std::vector<extent_type> M;
|
||||
std::vector<stride_type> ldAm;
|
||||
std::vector<stride_type> ldCm;
|
||||
for (char m : row_mode) {
|
||||
M.push_back(mode_size[m]);
|
||||
ldAm.push_back(mode_ldA[m]);
|
||||
ldCm.push_back(mode_ldC[m]);
|
||||
}
|
||||
|
||||
// Permute the col modes to promote coalescing
|
||||
// Sort the col modes by min(ldBn,ldCn) and tie-broken by ldBn
|
||||
std::sort(std::begin(col_mode), std::end(col_mode), [&](char n1, char n2) {
|
||||
return std::tie(std::min(mode_ldB[n1],mode_ldC[n1]),mode_ldB[n1])
|
||||
< std::tie(std::min(mode_ldB[n2],mode_ldC[n2]),mode_ldB[n2]);
|
||||
});
|
||||
// Compute sizes and strides of ordered col modes
|
||||
std::vector<extent_type> N;
|
||||
std::vector<stride_type> ldBn;
|
||||
std::vector<stride_type> ldCn;
|
||||
for (char n : col_mode) {
|
||||
N.push_back(mode_size[n]);
|
||||
ldBn.push_back(mode_ldB[n]);
|
||||
ldCn.push_back(mode_ldC[n]);
|
||||
}
|
||||
|
||||
if (parse_verbose) {
|
||||
std::cout << "C_";
|
||||
if (! row_mode.empty()) {
|
||||
std::cout << "(" << row_mode << ")";
|
||||
}
|
||||
if (! col_mode.empty()) {
|
||||
std::cout << "(" << col_mode << ")";
|
||||
}
|
||||
if (! bat_mode.empty()) {
|
||||
std::cout << "(" << bat_mode << ")";
|
||||
}
|
||||
std::cout << " = A_";
|
||||
if (! row_mode.empty()) {
|
||||
std::cout << "(" << row_mode << ")";
|
||||
}
|
||||
if (! red_mode.empty()) {
|
||||
std::cout << "(" << red_mode << ")";
|
||||
}
|
||||
if (! bat_mode.empty()) {
|
||||
std::cout << "(" << bat_mode << ")";
|
||||
}
|
||||
std::cout << " * B_";
|
||||
if (! col_mode.empty()) {
|
||||
std::cout << "(" << col_mode << ")";
|
||||
}
|
||||
if (! red_mode.empty()) {
|
||||
std::cout << "(" << red_mode << ")";
|
||||
}
|
||||
if (! bat_mode.empty()) {
|
||||
std::cout << "(" << bat_mode << ")";
|
||||
}
|
||||
std::cout << '\n';
|
||||
|
||||
int M_size = std::accumulate(std::begin(M), std::end(M), 1, std::multiplies<>{});
|
||||
int N_size = std::accumulate(std::begin(N), std::end(N), 1, std::multiplies<>{});
|
||||
int K_size = std::accumulate(std::begin(K), std::end(K), 1, std::multiplies<>{});
|
||||
int L_size = std::accumulate(std::begin(L), std::end(L), 1, std::multiplies<>{});
|
||||
|
||||
std::cout << " M : (" << M_size << ") ";
|
||||
for (char m : row_mode) std::cout << m << ":" << mode_size[m] << " ";
|
||||
std::cout << '\n';
|
||||
std::cout << " N : (" << N_size << ") ";
|
||||
for (char n : col_mode) std::cout << n << ":" << mode_size[n] << " ";
|
||||
std::cout << '\n';
|
||||
std::cout << " K : (" << K_size << ") ";
|
||||
for (char k : red_mode) std::cout << k << ":" << mode_size[k] << " ";
|
||||
std::cout << '\n';
|
||||
std::cout << " L : (" << L_size << ") ";
|
||||
for (char l : bat_mode) std::cout << l << ":" << mode_size[l] << " ";
|
||||
std::cout << '\n';
|
||||
|
||||
std::cout << " ldAm : " << ldAm << '\n';
|
||||
std::cout << " ldAk : " << ldAk << '\n';
|
||||
std::cout << " ldAl : " << ldAl << '\n';
|
||||
std::cout << " ldBn : " << ldBn << '\n';
|
||||
std::cout << " ldBk : " << ldBk << '\n';
|
||||
std::cout << " ldBl : " << ldBl << '\n';
|
||||
std::cout << " ldCm : " << ldCm << '\n';
|
||||
std::cout << " ldCn : " << ldCn << '\n';
|
||||
std::cout << " ldCl : " << ldCl << '\n';
|
||||
}
|
||||
|
||||
return {M, ldAm, ldCm,
|
||||
N, ldBn, ldCn,
|
||||
K, ldAk, ldBk,
|
||||
L, ldAl, ldBl, ldCl};
|
||||
}
|
||||
|
||||
static void
|
||||
print_usage() {
|
||||
std::cout <<
|
||||
"GETT problem command line parser:\n"
|
||||
" --modeA=<m0,...>\n"
|
||||
" A comma delimited list of characters that correspond to the row, reduction, and batch modes in A tensor.\n"
|
||||
" The semantic association of each symbolic mode is determined automatically.\n\n"
|
||||
|
||||
" --modeB=<m0,...>\n"
|
||||
" A comma delimited list of characters that correspond to the column, reduction, and batch modes in B tensor.\n"
|
||||
" The semantic association of each symbolic mode is determined automatically.\n\n"
|
||||
|
||||
" --modeC=<m0,...>\n"
|
||||
" A comma delimited list of characters that correspond to the row, column, and batch modes in B tensor.\n"
|
||||
" The semantic association of each symbolic mode is determined automatically.\n\n"
|
||||
|
||||
" --extents=<mode:extent,....>\n"
|
||||
" A command delimited list of symbolic mode and its corresponding extent.\n"
|
||||
" Extents are defaulted to 1 if any are not provided.\n\n"
|
||||
|
||||
"Example usage: gett.exe --modeC=m,n,l --modeA=m,k,l --modeB=k,n,l --extent=m:4096,n:4096,k:4096\n";
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace cutlass
|
||||
@@ -60,7 +60,7 @@ struct matrix_inf_norm_result {
|
||||
// and thus are best passed by reference or const reference.
|
||||
template <typename EngineType, typename LayoutType>
|
||||
matrix_inf_norm_result
|
||||
matrix_inf_norm(const cute::Tensor<EngineType, LayoutType>& host_matrix)
|
||||
matrix_inf_norm(cute::Tensor<EngineType, LayoutType> const& host_matrix)
|
||||
{
|
||||
using std::abs;
|
||||
using error_type = decltype(std::declval<matrix_inf_norm_result>().inf_norm);
|
||||
@@ -68,17 +68,14 @@ matrix_inf_norm(const cute::Tensor<EngineType, LayoutType>& host_matrix)
|
||||
error_type inf_norm = 0.0;
|
||||
bool found_nan = false;
|
||||
|
||||
const auto shape = host_matrix.shape();
|
||||
using index_type = std::decay_t<decltype(cute::get<0>(shape))>;
|
||||
// Computing the infinity norm requires that we be able
|
||||
// to treat the input as a matrix, with rows and columns.
|
||||
static_assert(std::is_integral_v<index_type>);
|
||||
const index_type num_rows = cute::get<0>(shape);
|
||||
const index_type num_cols = cute::get<1>(shape);
|
||||
const int64_t num_rows = cute::size<0>(host_matrix);
|
||||
const int64_t num_cols = cute::size<1>(host_matrix);
|
||||
|
||||
for(index_type i = 0; i < num_rows; ++i) {
|
||||
for(int64_t i = 0; i < num_rows; ++i) {
|
||||
error_type row_abs_sum = 0.0;
|
||||
for(index_type j = 0; j < num_cols; ++j) {
|
||||
for(int64_t j = 0; j < num_cols; ++j) {
|
||||
row_abs_sum += abs(host_matrix(i, j));
|
||||
}
|
||||
if(std::isnan(row_abs_sum)) {
|
||||
@@ -94,39 +91,27 @@ matrix_inf_norm(const cute::Tensor<EngineType, LayoutType>& host_matrix)
|
||||
// Infinity norm of (X - Y).
|
||||
template <typename EngineType, typename LayoutType>
|
||||
matrix_inf_norm_result
|
||||
matrix_diff_inf_norm(const cute::Tensor<EngineType, LayoutType>& X,
|
||||
const cute::Tensor<EngineType, LayoutType>& Y)
|
||||
matrix_diff_inf_norm(cute::Tensor<EngineType, LayoutType> const& X,
|
||||
cute::Tensor<EngineType, LayoutType> const& Y)
|
||||
{
|
||||
using std::abs;
|
||||
using error_type = decltype(std::declval<matrix_inf_norm_result>().inf_norm);
|
||||
|
||||
const auto X_shape = X.shape();
|
||||
const auto Y_shape = Y.shape();
|
||||
assert(cute::size<0>(X) == cute::size<0>(Y));
|
||||
assert(cute::size<1>(X) == cute::size<1>(Y));
|
||||
|
||||
using index_type = std::decay_t<decltype(cute::get<0>(X_shape))>;
|
||||
// Computing the infinity norm requires that we be able
|
||||
// to treat the input as a matrix, with rows and columns.
|
||||
static_assert(std::is_integral_v<index_type>);
|
||||
const index_type num_rows = cute::get<0>(X_shape);
|
||||
const index_type num_cols = cute::get<1>(X_shape);
|
||||
|
||||
assert(num_rows == cute::get<0>(Y_shape));
|
||||
assert(num_cols == cute::get<1>(Y_shape));
|
||||
|
||||
auto matrix_ij = [&](const auto& A, std::size_t i, std::size_t j) {
|
||||
return A(i, j);
|
||||
};
|
||||
auto diff_ij = [&](std::size_t i, std::size_t j) {
|
||||
return matrix_ij(X, i, j) - matrix_ij(Y, i, j);
|
||||
};
|
||||
const int64_t num_rows = cute::size<0>(X);
|
||||
const int64_t num_cols = cute::size<1>(X);
|
||||
|
||||
error_type inf_norm = 0.0;
|
||||
bool found_nan = false;
|
||||
|
||||
for(index_type i = 0; i < num_rows; ++i) {
|
||||
for(int64_t i = 0; i < num_rows; ++i) {
|
||||
error_type row_abs_sum = 0.0;
|
||||
for(index_type j = 0; j < num_cols; ++j) {
|
||||
row_abs_sum += abs(diff_ij(i, j));
|
||||
for(int64_t j = 0; j < num_cols; ++j) {
|
||||
row_abs_sum += abs(X(i,j) - Y(i,j));
|
||||
}
|
||||
if(std::isnan(row_abs_sum)) {
|
||||
found_nan = true;
|
||||
@@ -140,22 +125,22 @@ matrix_diff_inf_norm(const cute::Tensor<EngineType, LayoutType>& X,
|
||||
|
||||
template <typename EngineType_A, typename LayoutType_A,
|
||||
typename EngineType_B, typename LayoutType_B,
|
||||
typename EngineType_C_computed, typename LayoutType_C_computed,
|
||||
typename EngineType_C_expected, typename LayoutType_C_expected>
|
||||
typename EngineType_C, typename LayoutType_C,
|
||||
typename EngineType_C_ref, typename LayoutType_C_ref>
|
||||
void
|
||||
print_matrix_multiply_mollified_relative_error(
|
||||
const char A_value_type_name[],
|
||||
const cute::Tensor<EngineType_A, LayoutType_A>& A,
|
||||
const char B_value_type_name[],
|
||||
const cute::Tensor<EngineType_B, LayoutType_B>& B,
|
||||
const char C_value_type_name[],
|
||||
const cute::Tensor<EngineType_C_computed, LayoutType_C_computed>& C_computed,
|
||||
const cute::Tensor<EngineType_C_expected, LayoutType_C_expected>& C_expected)
|
||||
char const A_value_type_name[],
|
||||
cute::Tensor<EngineType_A, LayoutType_A> const& A,
|
||||
char const B_value_type_name[],
|
||||
cute::Tensor<EngineType_B, LayoutType_B> const& B,
|
||||
char const C_value_type_name[],
|
||||
cute::Tensor<EngineType_C, LayoutType_C> const& C,
|
||||
cute::Tensor<EngineType_C_ref, LayoutType_C_ref> const& C_ref)
|
||||
{
|
||||
const auto [A_norm, A_has_nan] = matrix_inf_norm(A);
|
||||
const auto [B_norm, B_has_nan] = matrix_inf_norm(B);
|
||||
const auto [C_norm, C_has_nan] = matrix_inf_norm(C_expected);
|
||||
const auto [diff_norm, diff_has_nan] = matrix_diff_inf_norm(C_computed, C_expected);
|
||||
const auto [C_norm, C_has_nan] = matrix_inf_norm(C_ref);
|
||||
const auto [diff_norm, diff_has_nan] = matrix_diff_inf_norm(C, C_ref);
|
||||
|
||||
const auto A_norm_times_B_norm = A_norm * B_norm;
|
||||
const auto relative_error = A_norm_times_B_norm == 0.0 ?
|
||||
@@ -164,18 +149,19 @@ print_matrix_multiply_mollified_relative_error(
|
||||
// For expected error bounds, please refer to the LAPACK Users' Guide,
|
||||
// in particular https://netlib.org/lapack/lug/node108.html .
|
||||
// Printing the infinity norm of C is a way to check
|
||||
// that both the function being tested (C_computed)
|
||||
// and the reference implementation (C_expected)
|
||||
// that both the function being tested (C)
|
||||
// and the reference implementation (C_ref)
|
||||
// don't just do nothing (or fill with zeros).
|
||||
using std::cout;
|
||||
cout << "Value type of A: " << A_value_type_name << '\n'
|
||||
using cute::shape;
|
||||
cout << "Matrix A: " << shape<0>(A) << "x" << shape<1>(A) << " of " << A_value_type_name << '\n'
|
||||
<< "Matrix B: " << shape<0>(B) << "x" << shape<1>(B) << " of " << B_value_type_name << '\n'
|
||||
<< "Matrix C: " << shape<0>(C) << "x" << shape<1>(C) << " of " << C_value_type_name << '\n'
|
||||
<< std::scientific
|
||||
<< "Infinity norm of A: " << A_norm << '\n'
|
||||
<< "Value type of B: " << B_value_type_name << '\n'
|
||||
<< "Infinity norm of B: " << B_norm << '\n'
|
||||
<< "Value type of C: " << C_value_type_name << '\n'
|
||||
<< "Infinity norm of C_expected: " << C_norm << '\n'
|
||||
<< "Infinity norm of (C_computed - C_expected): " << diff_norm << '\n';
|
||||
<< "Infinity norm of C: " << C_norm << '\n'
|
||||
<< "Infinity norm of (C - C_ref): " << diff_norm << '\n';
|
||||
|
||||
if(A_norm_times_B_norm == 0.0) {
|
||||
cout << "Mollified relative error: " << relative_error << '\n';
|
||||
@@ -183,11 +169,12 @@ print_matrix_multiply_mollified_relative_error(
|
||||
cout << "Relative error: " << relative_error << '\n';
|
||||
}
|
||||
|
||||
cout << "Did we encounter NaN in A? " << (A_has_nan ? "yes" : "no") << '\n'
|
||||
<< "Did we encounter NaN in B? " << (B_has_nan ? "yes" : "no") << '\n'
|
||||
<< "Did we encounter NaN in C_expected? " << (C_has_nan ? "yes" : "no") << '\n'
|
||||
<< "Did we encounter NaN in (C_computed - C_expected)? "
|
||||
<< (diff_has_nan ? "yes" : "no") << '\n';
|
||||
if (A_has_nan || B_has_nan || C_has_nan || diff_has_nan) {
|
||||
cout << "Did we encounter NaN in A? " << (A_has_nan ? "yes" : "no") << '\n'
|
||||
<< "Did we encounter NaN in B? " << (B_has_nan ? "yes" : "no") << '\n'
|
||||
<< "Did we encounter NaN in C? " << (C_has_nan ? "yes" : "no") << '\n'
|
||||
<< "Did we encounter NaN in (C - C_ref)? " << (diff_has_nan ? "yes" : "no") << '\n';
|
||||
}
|
||||
}
|
||||
|
||||
template <typename EngineType, typename LayoutType>
|
||||
@@ -233,3 +220,70 @@ auto host_matrix_to_const_cute_tensor(CutlassHostTensorType& X)
|
||||
auto X_data_const = const_cast<std::add_const_t< decltype(X_data)> >(X_data);
|
||||
return cute::make_tensor(X_data_const, layout);
|
||||
};
|
||||
|
||||
|
||||
template <typename T1, typename T2>
|
||||
double
|
||||
print_relative_error(
|
||||
std::size_t n,
|
||||
T1 const& data,
|
||||
T2 const& reference,
|
||||
bool print_verbose = false,
|
||||
bool print_error = true) {
|
||||
using std::abs; using std::sqrt;
|
||||
|
||||
// Use either double or complex<double> for error computation
|
||||
using value_type = cute::remove_cvref_t<decltype(reference[0])>;
|
||||
using error_type = std::conditional_t<cute::is_complex<value_type>::value,
|
||||
cute::complex<double>,
|
||||
double>;
|
||||
|
||||
if (print_verbose) {
|
||||
std::cout << "Idx:\t"<< "Val\t" << "RefVal\t" << "RelError" << std::endl;
|
||||
}
|
||||
|
||||
double eps = 1e-200;
|
||||
|
||||
double tot_error_sq = 0;
|
||||
double tot_norm_sq = 0;
|
||||
double tot_ind_rel_err = 0;
|
||||
double max_ind_rel_err = 0;
|
||||
for (std::size_t i = 0; i < n; ++i)
|
||||
{
|
||||
error_type val = data[i];
|
||||
error_type ref = reference[i];
|
||||
|
||||
double aref = abs(ref);
|
||||
double diff = abs(ref - val);
|
||||
double rel_error = diff / (aref + eps);
|
||||
|
||||
// Individual relative error
|
||||
tot_ind_rel_err += rel_error;
|
||||
|
||||
// Maximum relative error
|
||||
max_ind_rel_err = std::max(max_ind_rel_err, rel_error);
|
||||
|
||||
// Total relative error
|
||||
tot_error_sq += diff * diff;
|
||||
tot_norm_sq += aref * aref;
|
||||
|
||||
if (print_verbose) {
|
||||
std::cout << i << ":\t" << val << "\t" << ref << "\t" << rel_error << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
printf("Vector reference norm: [%.5e]\n", sqrt(tot_norm_sq));
|
||||
|
||||
double tot_rel_err = sqrt(tot_error_sq/(tot_norm_sq+eps));
|
||||
if (print_error)
|
||||
printf("Vector relative error: [%.5e]\n", tot_rel_err);
|
||||
|
||||
double ave_rel_err = tot_ind_rel_err / double(n);
|
||||
if (print_error)
|
||||
printf("Average relative error: [%.5e]\n", ave_rel_err);
|
||||
|
||||
if (print_error)
|
||||
printf("Maximum relative error: [%.5e]\n", max_ind_rel_err);
|
||||
|
||||
return tot_rel_err;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief GETT device reference code
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
|
||||
namespace cutlass::reference::device {
|
||||
|
||||
template <
|
||||
class ATensor,
|
||||
class BTensor,
|
||||
class CTensor,
|
||||
class DTensor,
|
||||
class ElementAccumulator,
|
||||
class ElementEpilogue>
|
||||
__global__ static
|
||||
void
|
||||
gett_kernel(
|
||||
DTensor D,
|
||||
ATensor const A,
|
||||
BTensor const B,
|
||||
CTensor const C,
|
||||
ElementEpilogue alpha, ElementEpilogue beta,
|
||||
ElementAccumulator acc_init)
|
||||
{
|
||||
using namespace cute;
|
||||
|
||||
static_assert(DTensor::rank == 3, "(M,N,L)");
|
||||
static_assert(ATensor::rank == 3, "(M,K,L)");
|
||||
static_assert(BTensor::rank == 3, "(N,K,L)");
|
||||
static_assert(CTensor::rank == 3, "(M,N,L)");
|
||||
|
||||
assert(size<0>(A) == size<0>(D)); // M
|
||||
assert(size<0>(C) == size<0>(D)); // M
|
||||
assert(size<0>(B) == size<1>(D)); // N
|
||||
assert(size<1>(C) == size<1>(D)); // N
|
||||
assert(size<1>(A) == size<1>(B)); // K
|
||||
assert(size<2>(A) == size<2>(D)); // L
|
||||
assert(size<2>(B) == size<2>(D)); // L
|
||||
assert(size<2>(C) == size<2>(D)); // L
|
||||
|
||||
NumericConverter<ElementAccumulator, typename ATensor::value_type> a_converter;
|
||||
NumericConverter<ElementAccumulator, typename BTensor::value_type> b_converter;
|
||||
NumericConverter<ElementEpilogue, ElementAccumulator> acc_converter;
|
||||
NumericConverter<ElementEpilogue, typename CTensor::value_type> source_converter;
|
||||
NumericConverter<typename DTensor::value_type, ElementEpilogue> output_converter;
|
||||
|
||||
// Thread id to each element of D
|
||||
for (int tid = threadIdx.x + blockDim.x * blockIdx.x;
|
||||
tid < size(D);
|
||||
tid += blockDim.x * gridDim.x) {
|
||||
// (m,n,l) coordinate
|
||||
auto mnl_coord = idx2crd(tid, product_each(shape(D)));
|
||||
auto m = get<0>(mnl_coord);
|
||||
auto n = get<1>(mnl_coord);
|
||||
auto l = get<2>(mnl_coord);
|
||||
|
||||
auto A_ml = A(m,_,l);
|
||||
auto B_nl = B(n,_,l);
|
||||
|
||||
ElementAccumulator accum = ElementAccumulator(0);
|
||||
for (int k = 0; k < size<1>(A); ++k) {
|
||||
ElementAccumulator a = a_converter(A_ml(k));
|
||||
ElementAccumulator b = b_converter(B_nl(k));
|
||||
accum += a * b;
|
||||
}
|
||||
|
||||
ElementEpilogue scaled_output = (alpha * acc_converter(accum)) + (beta * source_converter(C(m,n,l)));
|
||||
D(m,n,l) = output_converter(scaled_output);
|
||||
}
|
||||
}
|
||||
|
||||
// Most general version
|
||||
template <
|
||||
class ProblemShapeMNKL,
|
||||
class ElementA,
|
||||
class StrideA,
|
||||
class ElementB,
|
||||
class StrideB,
|
||||
class ElementAccumulator,
|
||||
class ElementC,
|
||||
class StrideC,
|
||||
class ElementD,
|
||||
class StrideD,
|
||||
class ElementEpilogue>
|
||||
void
|
||||
gett(
|
||||
ProblemShapeMNKL problem_shape_mnkl,
|
||||
ElementA const* ptr_A, StrideA stride_a_mkl,
|
||||
ElementB const* ptr_B, StrideB stride_b_nkl,
|
||||
ElementAccumulator _,
|
||||
ElementC const* ptr_C, StrideC stride_c_mnl,
|
||||
ElementD * ptr_D, StrideD stride_d_mnl,
|
||||
ElementEpilogue alpha, ElementEpilogue beta,
|
||||
cudaStream_t stream = 0) {
|
||||
using namespace cute;
|
||||
|
||||
static_assert(rank(ProblemShapeMNKL{}) == 4);
|
||||
auto M = get<0>(problem_shape_mnkl);
|
||||
auto N = get<1>(problem_shape_mnkl);
|
||||
auto K = get<2>(problem_shape_mnkl);
|
||||
auto L = get<3>(problem_shape_mnkl);
|
||||
|
||||
// Represent the full tensors
|
||||
auto A = make_tensor(make_gmem_ptr(ptr_A), make_shape(M,K,L), stride_a_mkl); // (M,K,L)
|
||||
auto B = make_tensor(make_gmem_ptr(ptr_B), make_shape(N,K,L), stride_b_nkl); // (N,K,L)
|
||||
auto C = make_tensor(make_gmem_ptr(ptr_C), make_shape(M,N,L), stride_c_mnl); // (M,N,L)
|
||||
auto D = make_tensor(make_gmem_ptr(ptr_D), make_shape(M,N,L), stride_d_mnl); // (M,N,L)
|
||||
|
||||
dim3 dimBlock(256);
|
||||
dim3 dimGrid(240);
|
||||
gett_kernel<<< dimGrid, dimBlock, 0, stream >>>(D, A, B, C, alpha, beta, ElementAccumulator(0));
|
||||
}
|
||||
|
||||
} // namespace cutlass::reference::device
|
||||
Reference in New Issue
Block a user