leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 25 Packages Projects Releases Wiki Activity

Checkpointing CUTLASS 1.1 release.

Browse Source
This commit is contained in:
akerr
2018-09-18 16:58:03 -07:00
parent cf0301e00f
commit 461f417b9d
193 changed files with 29495 additions and 4770 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/util/command_line.h
+50 -23
View File
@@ -108,7 +108,7 @@ struct CommandLine {
}
/**
* Returns the commandline parameter for a given index (not including flags)
* Returns the boolean value specified for a given commandline parameter --<flag>=<bool>
*/
void get_cmd_line_argument(const char* arg_name, bool& val, bool _default = true) const {
val = _default;
@@ -156,27 +156,7 @@ struct CommandLine {
for (int i = 0; i < keys.size(); ++i) {
if (keys[i] == string(arg_name)) {
string val_string(values[i]);
istringstream str_stream(val_string);
string::size_type old_pos = 0;
string::size_type new_pos = 0;
// Iterate <sep>-delimited values
value_t val;
while ((new_pos = val_string.find(sep, old_pos)) != string::npos) {
if (new_pos != old_pos) {
str_stream.width(new_pos - old_pos);
str_stream >> val;
vals.push_back(val);
}
// skip over delimiter
str_stream.ignore(1);
old_pos = new_pos + 1;
}
// Read last value
str_stream >> val;
vals.push_back(val);
seperate_string(val_string, vals, sep);
}
}
}
@@ -184,7 +164,7 @@ struct CommandLine {
/**
* Returns the values specified for a given commandline parameter
* --<flag>=<key:value>,<key:value>*
* --<flag>=<value>,<value_start:value_end>*
*/
void get_cmd_line_argument_pairs(const char* arg_name,
std::vector<std::pair<std::string, std::string> >& tokens,
@@ -198,6 +178,26 @@ struct CommandLine {
}
}
/**
* Returns a list of ranges specified for a given commandline parameter
* --<flag>=<key:value>,<key:value>*
*/
void get_cmd_line_argument_ranges(const char* arg_name,
std::vector<std::vector<std::string> >& vals,
char delim = ',',
char sep = ':') const {
std::vector<std::string> ranges;
get_cmd_line_arguments(arg_name, ranges, delim);
for (std::vector<std::string>::const_iterator range = ranges.begin();
range != ranges.end(); ++range) {
std::vector<std::string> range_vals;
seperate_string(*range, range_vals, sep);
vals.push_back(range_vals);
}
}
/**
* The number of pairs parsed
*/
@@ -249,6 +249,33 @@ struct CommandLine {
tokens.push_back(tok->first);
}
}
template <typename value_t>
static void seperate_string(std::string const& str,
std::vector<value_t>& vals,
char sep = ',') {
std::istringstream str_stream(str);
std::string::size_type old_pos = 0;
std::string::size_type new_pos = 0;
// Iterate <sep>-delimited values
value_t val;
while ((new_pos = str.find(sep, old_pos)) != std::string::npos) {
if (new_pos != old_pos) {
str_stream.width(new_pos - old_pos);
str_stream >> val;
vals.push_back(val);
}
// skip over delimiter
str_stream.ignore(1);
old_pos = new_pos + 1;
}
// Read last value
str_stream >> val;
vals.push_back(val);
}
};
} // namespace cutlass
tools/util/device_memory.h
+23 -4
View File
@@ -26,9 +26,9 @@
#include <memory>
#include <cutlass/util/debug.h>
#include <cutlass/util/platform.h>
#include <tools/util/exceptions.h>
#include "cutlass/util/debug.h"
#include "cutlass/util/platform.h"
#include "tools/util/exceptions.h"
namespace cutlass {
namespace device_memory {
@@ -124,6 +124,10 @@ struct allocation {
}
};
//
// Data members
//
/// Number of elements of T allocated on the current CUDA device
size_t capacity;
@@ -131,7 +135,7 @@ struct allocation {
platform::unique_ptr<T, deleter> smart_ptr;
//
//
// Methods
//
/// Constructor: allocates no memory
@@ -140,6 +144,11 @@ struct allocation {
/// Constructor: allocates \p capacity elements on the current CUDA device
allocation(size_t _capacity) : smart_ptr(allocate<T>(_capacity)), capacity(_capacity) {}
/// Copy constructor
allocation(allocation const &p): smart_ptr(allocate<T>(p.capacity)), capacity(p.capacity) {
copy_device_to_device(smart_ptr.get(), p.get(), capacity);
}
/// Destructor
~allocation() { reset(); }
@@ -172,6 +181,16 @@ struct allocation {
/// Returns the deleter object which would be used for destruction of the managed object (const)
const deleter& get_deleter() const { return smart_ptr.get_deleter(); }
/// Copies a device-side memory allocation
allocation & operator=(allocation const &p) {
if (capacity != p.capacity) {
smart_ptr.reset(allocate<T>(p.capacity));
capacity = p.capacity;
}
copy_device_to_device(smart_ptr.get(), p.get(), capacity);
return *this;
}
};
} // namespace device_memory
tools/util/distribution.h
+138
View File
@@ -0,0 +1,138 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
/*! \file
\brief This header contains a class to parametrize a statistical distribution function.
*/
#include <fstream>
namespace cutlass {
////////////////////////////////////////////////////////////////////////////////////////////////////
/// Distribution type
struct Distribution {
/// Variant types
enum Kind { Invalid, Uniform, Gaussian, Linear, Identity };
/// Distribution state
union {
/// Uniform distribution
struct {
double min;
double max;
} uniform;
/// Gaussian distribution
struct {
double mean;
double stddev;
} gaussian;
/// Elements are linear combination of row and column index
struct {
double offset;
double delta_row;
double delta_column;
} linear;
};
/// Active variant kind
Kind kind;
/// Random values are cast to integer after scaling by this power of two
int int_scale;
//
// Methods
//
Distribution() : kind(Invalid), int_scale(0) {}
/// Configures distribution as uniform random
Distribution &set_uniform(double _min, double _max, int _int_scale = 0) {
kind = Uniform;
uniform.min = _min;
uniform.max = _max;
int_scale = _int_scale;
return *this;
}
/// Configures distribution as Gaussian distribution
Distribution &set_gaussian(double _mean, double _stddev, int _int_scale = 0) {
kind = Gaussian;
gaussian.mean = _mean;
gaussian.stddev = _stddev;
int_scale = _int_scale;
return *this;
}
/// Sets identity
Distribution &set_identity() {
kind = Identity;
return *this;
}
/// Configures distribution as linear combination of row and column index
Distribution &set_linear(double _offset, double _delta_row, double _delta_column) {
kind = Linear;
linear.offset = _offset;
linear.delta_row = _delta_row;
linear.delta_column = _delta_column;
return *this;
}
};
} // namespace cutlass
////////////////////////////////////////////////////////////////////////////////////////////////////
/// Prints a Distribution to ostream
inline std::ostream &operator<<(std::ostream &out, cutlass::Distribution const &dist) {
switch (dist.kind) {
case cutlass::Distribution::Uniform:
out << "uniform, min: " << dist.uniform.min << ", max: " << dist.uniform.max;
break;
case cutlass::Distribution::Gaussian:
out << "gaussian, mean: " << dist.gaussian.mean << ", stddev: " << dist.gaussian.stddev;
break;
case cutlass::Distribution::Linear:
out << "linear, mean: " << dist.linear.offset << ", delta_row: " << dist.linear.delta_row
<< ", delta_column: " << dist.linear.delta_column;
break;
case cutlass::Distribution::Identity:
break;
default:
out << "unknown";
}
out << ", int_scale: " << dist.int_scale;
return out;
}
////////////////////////////////////////////////////////////////////////////////////////////////////
tools/util/exceptions.h
+1 -1
View File
@@ -28,7 +28,7 @@
#include <iosfwd>
#include <stdexcept>
#include <cutlass/util/platform.h>
#include "cutlass/util/platform.h"
namespace cutlass {
tools/util/half.h
+27 -30
View File
@@ -107,6 +107,33 @@ class half_t {
uint16_t& raw() { return x; }
uint16_t raw() const { return x; }
//
// Stream interactions
//
/// put to stream - half_t-precision types bitcast as unsigned shorts if base is hexadecimal
friend std::ostream& operator<<(std::ostream& out, cutlass::half_t const& h) {
if (out.flags() & std::ios::hex) {
return out << h.x;
} else {
return out << float(h);
}
}
/// read from stream - half_t-precision types parsed as unsigned shorts if base is hexadecimal
friend std::istream& operator>>(std::istream& in, cutlass::half_t& h) {
if (in.flags() & std::ios::hex) {
unsigned short u = 0;
in >> u;
h = cutlass::half_t::bitcast(u);
} else {
float f = 0;
in >> f;
h = cutlass::half_t(f);
}
return in;
}
public:
/// data
unsigned short x;
@@ -167,9 +194,6 @@ cutlass::half_t operator-(float, cutlass::half_t const&);
cutlass::half_t operator*(float, cutlass::half_t const&);
cutlass::half_t operator/(float, cutlass::half_t const&);
std::ostream& operator<<(std::ostream&, cutlass::half_t const&); /// writes a half_t
std::istream& operator>>(std::istream&, cutlass::half_t&); /// reads a half_t
#ifdef BOOST_LEXICAL_CAST_INCLUDED
namespace boost {
@@ -714,30 +738,3 @@ inline cutlass::half_t sqrt(cutlass::half_t const& h) {
return cutlass::half_t(std::sqrt(float(h)));
}
} // namespace std
//
// Stream interactions
//
/// put to stream - half_t-precision types bitcast as unsigned shorts if base is hexadecimal
inline std::ostream& operator<<(std::ostream& out, cutlass::half_t const& h) {
if (out.flags() & std::ios::hex) {
return out << h.x;
} else {
return out << float(h);
}
}
/// read from stream - half_t-precision types parsed as unsigned shorts if base is hexadecimal
inline std::istream& operator>>(std::istream& in, cutlass::half_t& h) {
if (in.flags() & std::ios::hex) {
unsigned short u = 0;
in >> u;
h = cutlass::half_t::bitcast(u);
} else {
float f = 0;
in >> f;
h = cutlass::half_t(f);
}
return in;
}
tools/util/host_matrix.h
+264
View File
@@ -0,0 +1,264 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
/*! \file
\brief HostMatrix is a helper to define a HostTensor of rank=2 with a contiguous layout.
See tools/util/host_tensor.h for more details.
*/
#include "cutlass/matrix_traits.h"
#include "tools/util/host_tensor.h"
#include "tools/util/reference/host/gemm.h"
namespace cutlass {
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Helper to define a rank=2 host matrix with contiguous layout
template <
typename T
>
class HostMatrix :
public HostTensor<T, 2, MatrixLayout::ContiguousLayout, 3, int, long long> {
public:
/// Base class is a HostTensor of rank=2 with contiguous layout
typedef HostTensor<T, 2, MatrixLayout::ContiguousLayout, 3, int, long long> Base;
/// Index type
typedef typename Base::Index Index;
private:
/// Layout of contiguous matrix
MatrixLayout::Kind layout_;
public:
/// Default ctor
HostMatrix(): layout_(MatrixLayout::kColumnMajor) { }
/// Constructs a HostTensor from size. Assumes column-major and infers leading dimension
HostMatrix(MatrixCoord const& size, bool _device_backed = true): layout_(MatrixLayout::kColumnMajor) {
Index ldm = size[0];
this->reset(MatrixLayout::ContiguousLayout::stride(layout_, ldm), size, _device_backed);
}
/// Constructs a HostTensor from size and layout - infers leading dimension
HostMatrix(MatrixCoord const& size, MatrixLayout::Kind layout, bool _device_backed = true): layout_(layout) {
Index ldm = (layout_ == MatrixLayout::kColumnMajor ? size[0] : size[1]);
this->reset(MatrixLayout::ContiguousLayout::stride(layout_, ldm), size, _device_backed);
}
/// Constructs a HostTensor given size, layout, and leading dimension
HostMatrix(MatrixCoord const& size, Index ldm, MatrixLayout::Kind layout, bool _device_backed = true): layout_(layout) {
this->reset(MatrixLayout::ContiguousLayout::stride(layout_, ldm), size, _device_backed);
}
/// Returns contiguous matrix layout kind
MatrixLayout::Kind get_layout() const {
return layout_;
}
/// Resizes a matrix
void resize(MatrixCoord const &_size, MatrixLayout::Kind layout, Index ldm = 0, bool _device_backed = true) {
if (!ldm) {
ldm = (layout == MatrixLayout::kColumnMajor ? _size[0] : _size[1]);
}
layout_ = layout;
this->reset(MatrixLayout::ContiguousLayout::stride(layout_, ldm), _size, _device_backed);
}
/// Helper to resize matrix
void resize(Index rows, Index columns, MatrixLayout::Kind layout, Index ldm = 0, bool _device_backed = true) {
this->resize(MatrixCoord(rows, columns), layout, ldm,_device_backed);
}
/// Helper to resize matrix
void resize_matrix(Index rows, Index columns, MatrixLayout::Kind layout, Index ldm = 0, bool _device_backed = true) {
this->resize(MatrixCoord(rows, columns), layout, ldm,_device_backed);
}
/// Gets the leading dimension of the matrix
Index leading_dim() const {
if (layout_ == MatrixLayout::kColumnMajor) {
return this->stride(MatrixLayout::ContiguousLayout::kColumn);
}
else {
return this->stride(MatrixLayout::ContiguousLayout::kRow);
}
}
/// Returns size as a MatrixCoord
MatrixCoord size() const {
return MatrixCoord(Base::size());
}
/// Returns size in the given dimension
Index size(int idx) const {
return Base::size(idx);
}
/// Helper to call GEMM operation on HostMatrix objects that differ only in their scalar type.
template <typename A, typename B, typename Ctype, typename Stype>
void gemm(
HostMatrix<A> const& tensor_a,
HostMatrix<B> const& tensor_b,
Stype alpha = Stype(1),
Stype beta = Stype(0)) {
gemm::GemmCoord problem_size(
tensor_a.size().column(),
this->size().column(),
this->size().row(),
1);
cutlass::reference::host::Gemm(
problem_size,
alpha,
tensor_a,
tensor_b,
beta,
*this,
Ctype(0));
}
};
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Helper to define a rank=2 host matrix with column-major layout
template <
typename T
>
class HostMatrixColumnMajor :
public HostTensor<T, 2, MatrixLayout::ColumnMajor, 2, int, long long> {
public:
/// Base class is a HostTensor of rank=2 with contiguous layout
typedef HostTensor<T, 2, MatrixLayout::ColumnMajor, 2, int, long long> Base;
/// Tensor coordinate
typedef typename Base::TensorCoord TensorCoord;
/// Index type
typedef typename Base::Index Index;
public:
/// Default ctor
HostMatrixColumnMajor() { }
/// Constructs a HostMatrixColumnMajor from size. Assumes column-major and infers leading dimension
HostMatrixColumnMajor(TensorCoord const& size, bool _device_backed = true): Base(size, size[0], _device_backed) {
}
/// Constructs a HostMatrixColumnMajor given size, layout, and leading dimension
HostMatrixColumnMajor(TensorCoord const& size, Index ldm, bool _device_backed = true) {
this->reset(make_Coord(ldm, 1), size, _device_backed);
}
/// Resizes a matrix
void resize(MatrixCoord const &size, int ldm = 0, bool _device_backed = true) {
this->reset(ldm, size, _device_backed);
}
/// Returns contiguous matrix layout kind
MatrixLayout::Kind get_layout() const {
return MatrixLayout::kColumnMajor;
}
/// Gets the leading dimension of the matrix
Index leading_dim() const {
return this->stride(0);
}
/// Returns size as a MatrixCoord
MatrixCoord size() const {
return MatrixCoord(Base::size());
}
};
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Helper to define a rank=2 host matrix with row-major layout
template <
typename T
>
class HostMatrixRowMajor :
public HostTensor<T, 2, MatrixLayout::RowMajor, 2, int, long long> {
public:
/// Base class is a HostTensor of rank=2 with contiguous layout
typedef HostTensor<T, 2, MatrixLayout::RowMajor, 2, int, long long> Base;
/// Tensor coordinate
typedef typename Base::TensorCoord TensorCoord;
/// Index type
typedef typename Base::Index Index;
public:
/// Default ctor
HostMatrixRowMajor() { }
/// Constructs a HostTensor from size. Assumes column-major and infers leading dimension
HostMatrixRowMajor(TensorCoord const& size, bool _device_backed = true) {
this->reset(make_Coord(size[1], 1), size, _device_backed);
}
/// Constructs a HostTensor given size, layout, and leading dimension
HostMatrixRowMajor(TensorCoord const& size, Index ldm, bool _device_backed = true) {
this->reset(make_Coord(ldm, 1), size, _device_backed);
}
/// Resizes a matrix
void resize(MatrixCoord const &size, int ldm = 0, bool _device_backed = true) {
this->reset(ldm, size, _device_backed);
}
/// Returns contiguous matrix layout kind
MatrixLayout::Kind get_layout() const {
return MatrixLayout::kRowMajor;
}
/// Gets the leading dimension of the matrix
Index leading_dim() const {
return this->stride(0);
}
/// Returns size as a MatrixCoord
MatrixCoord size() const {
return MatrixCoord(Base::size());
}
};
///////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace cutlass
tools/util/host_matrix_view.h
+205
View File
@@ -0,0 +1,205 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
/*! \file
\brief HostMatrix is a helper to define a HostTensor of rank=2 with a contiguous layout.
See tools/util/host_tensor.h for more details.
*/
#include "cutlass/matrix_traits.h"
#include "tools/util/host_tensor.h"
#include "tools/util/reference/host/gemm.h"
namespace cutlass {
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Helper to define a rank=2 host matrix with contiguous layout
template <
typename T
>
class HostMatrixView :
public HostTensorView<T, 2, MatrixLayout::ContiguousLayout, 3, int> {
public:
/// Base class is a HostTensor of rank=2 with contiguous layout
typedef HostTensorView<T, 2, MatrixLayout::ContiguousLayout, 3, int> Base;
/// Tensor coordinate
typedef typename Base::TensorCoord TensorCoord;
/// Index type
typedef typename Base::Index Index;
private:
/// Layout of contiguous matrix
MatrixLayout::Kind layout_;
public:
/// Default ctor
HostMatrixView(): layout_(MatrixLayout::kColumnMajor) { }
/// Constructs a HostTensor from size. Assumes column-major and infers leading dimension
HostMatrixView(TensorCoord const& size): layout_(MatrixLayout::kColumnMajor) {
Index ldm = size[0];
this->reset(MatrixLayout::ContiguousLayout::stride(layout_, ldm), size);
}
/// Constructs a HostTensor from size and layout - infers leading dimension
HostMatrixView(TensorCoord const& size, MatrixLayout::Kind layout): layout_(layout) {
Index ldm = (layout_ == MatrixLayout::kColumnMajor ? size[0] : size[1]);
this->reset(MatrixLayout::ContiguousLayout::stride(layout_, ldm), size);
}
/// Constructs a HostTensor given size, layout, and leading dimension
HostMatrixView(TensorCoord const& size, Index ldm, MatrixLayout::Kind layout): layout_(layout) {
this->reset(MatrixLayout::ContiguousLayout::stride(layout_, ldm), size);
}
/// Gets the leading dimension of the matrix
Index leading_dim() const {
if (layout_ == MatrixLayout::kColumnMajor) {
return this->stride(MatrixLayout::ContiguousLayout::kColumn);
}
else {
return this->stride(MatrixLayout::ContiguousLayout::kRow);
}
}
/// Returns contiguous matrix layout kind
MatrixLayout::Kind get_layout() const {
return layout_;
}
/// Returns size as a MatrixCoord
MatrixCoord size() const {
return MatrixCoord(Base::size());
}
};
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Helper to define a rank=2 host matrix with column-major layout
template <typename T>
class HostMatrixViewColumnMajor :
public HostTensorView<T, 2, MatrixLayout::ColumnMajor, 2, int, long long> {
public:
/// Base class is a HostTensorView of rank=2 with contiguous layout
typedef HostTensorView<T, 2, MatrixLayout::ColumnMajor, 2, int, long long> Base;
/// Tensor coordinate
typedef typename Base::TensorCoord TensorCoord;
/// Index type
typedef typename Base::Index Index;
public:
/// Default ctor
HostMatrixViewColumnMajor() { }
/// Constructs a HostMatrixViewColumnMajor from size. Assumes column-major and infers leading dimension
HostMatrixViewColumnMajor(TensorCoord const& size): Base(size, size[0]) {
}
/// Constructs a HostMatrixViewColumnMajor given size, layout, and leading dimension
HostMatrixViewColumnMajor(TensorCoord const& size, Index ldm) {
this->reset(make_Coord(ldm, 1), size);
}
/// Returns contiguous matrix layout kind
MatrixLayout::Kind get_layout() const {
return MatrixLayout::kColumnMajor;
}
/// Gets the leading dimension of the matrix
Index leading_dim() const {
return this->stride(0);
}
/// Returns size as a MatrixCoord
MatrixCoord size() const {
return MatrixCoord(Base::size());
}
};
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Helper to define a rank=2 host matrix with row-major layout
template <typename T>
class HostMatrixViewRowMajor :
public HostTensorView<T, 2, MatrixLayout::RowMajor, 2, int, long long> {
public:
/// Base class is a HostTensor of rank=2 with contiguous layout
typedef HostTensorView<T, 2, MatrixLayout::RowMajor, 2, int, long long> Base;
/// Tensor coordinate
typedef typename Base::TensorCoord TensorCoord;
/// Index type
typedef typename Base::Index Index;
public:
/// Default ctor
HostMatrixViewRowMajor() { }
/// Constructs a HostMatrixViewRowMajor from size. Assumes column-major and infers leading dimension
HostMatrixViewRowMajor(TensorCoord const& size): Base(size, size[1]) {
}
/// Constructs a HostMatrixViewRowMajor given size, layout, and leading dimension
HostMatrixViewRowMajor(TensorCoord const& size, Index ldm) {
this->reset(make_Coord(ldm, 1), size);
}
/// Returns contiguous matrix layout kind
MatrixLayout::Kind get_layout() const {
return MatrixLayout::kRowMajor;
}
/// Gets the leading dimension of the matrix
Index leading_dim() const {
return this->stride(0);
}
/// Returns size as a MatrixCoord
MatrixCoord size() const {
return MatrixCoord(Base::size());
}
};
///////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace cutlass
tools/util/host_tensor.h
+215 -192
View File
@@ -25,51 +25,126 @@
#pragma once
/*! \file
\brief Template class to perform computations on tensors and manage memory.
\brief HostTensor contributes management for both host and device memory.
HostTensor allocates host and device memory upon construction. Basic element-wise operations on
host memory synchronize device memory automatically. Explicit copy operations provide abstractions
for CUDA memcpy operations.
Call device_{data, ref, view} for accessing device memory allocations.
See cutlass/tensor_ref.h, cutlass/tensor_view.h, and tools/util/host_tensor_view.h for more details.
*/
#include <cutlass/cutlass.h>
#include <cutlass/matrix_traits.h>
#include <tools/util/device_memory.h>
#include <tools/util/host_tensor_view.h>
#include <tools/util/type_traits.h>
#include "cutlass/cutlass.h"
#include "cutlass/matrix_traits.h"
#include "cutlass/tensor_ref.h"
#include "tools/util/device_memory.h"
#include "tools/util/host_tensor_view.h"
#include "tools/util/type_traits.h"
#include <vector>
namespace cutlass {
template <typename T, bool DeviceBacked_ = true>
class HostTensor : public HostTensorView<T> {
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Host tensor
template <
/// Scalar data type (may be mapped to compatible types for use on host and device)
typename T,
/// Rank of logical tensor
int Rank_ = 4,
/// Maps a Coord<Rank_> in the logical tensor index space to the internal n-D array
typename MapFunc_ = IdentityTensorMapFunc<Rank_>,
/// Rank of internal n-D array
int StorageRank_ = MapFunc_::kStorageRank,
/// Index type used for coordinates
typename Index_ = int,
/// Index type used for offsets and pointer differences
typename LongIndex_ = long long
>
class HostTensor : public HostTensorView<
typename TypeTraits<T>::host_type,
Rank_,
MapFunc_,
StorageRank_,
Index_,
LongIndex_> {
public:
/// Type used for host-side allocations
typedef typename TypeTraits<T>::host_type HostType;
/// Type used for device-side allocations
typedef typename TypeTraits<T>::device_type DeviceType;
/// Base class
typedef HostTensorView<T> Base;
/// If true, allocates device side memory
static bool const DeviceBacked = DeviceBacked_;
/// Rank of tensor
static int const Rank = Base::Rank;
typedef HostTensorView<
typename TypeTraits<T>::host_type,
Rank_,
MapFunc_,
StorageRank_,
Index_,
LongIndex_> Base;
/// Type used to compute the offset of an element to the base of a tensor
typedef typename Base::Offset_t Offset_t;
/// Tensor reference to host memory
typedef typename Base::TensorRef_t TensorRef_t;
typedef LongIndex_ LongIndex;
/// Tensor reference to device memory
typedef TensorRef<DeviceType, TensorRef_t::Rank> DeviceTensorRef;
typedef typename cutlass::TensorRef<
DeviceType,
Rank_,
MapFunc_,
StorageRank_,
Index_,
LongIndex_> DeviceTensorRef;
/// Tensor reference to constant device memory
typedef TensorRef<DeviceType const, TensorRef_t::Rank> ConstDeviceTensorRef;
typedef typename DeviceTensorRef::ConstTensorRef ConstDeviceTensorRef;
/// Coordinate into tensor
typedef typename Base::Coord_t Coord_t;
/// TensorView to device memory
typedef TensorView<
DeviceType,
Rank_,
MapFunc_,
StorageRank_,
Index_,
LongIndex_> DeviceTensorView;
/// Tensor reference to constant device memory
typedef typename DeviceTensorView::ConstTensorView ConstDeviceTensorView;
/// Tensor reference to host memory
typedef typename Base::TensorRef TensorRef;
/// Tensor view to host memory
typedef TensorView<
typename TypeTraits<T>::host_type,
Rank_,
MapFunc_,
StorageRank_,
Index_,
LongIndex_> HostTensorView;
/// Tensor view to host memory
typedef typename HostTensorView::ConstTensorView ConstHostTensorView;
/// Coordinate in logical tensor space
typedef typename TensorRef::TensorCoord TensorCoord;
/// Coordinate in storage n-D array
typedef typename TensorRef::StorageCoord StorageCoord;
/// Stride vector in storage coordinate space
/// Least significant stride is = 1 and not stored
typedef typename TensorRef::StrideVector StrideVector;
/// Rank of internal storage.
static int const kStorageRank = Base::kStorageRank;
private:
/// Host-side memory allocation
std::vector<T> host_;
std::vector<HostType> host_;
/// Device-side memory
cutlass::device_memory::allocation<DeviceType> device_;
@@ -82,232 +157,173 @@ class HostTensor : public HostTensorView<T> {
/// Default constructor
HostTensor() {}
/// Constructs a Tensor_view from stride and size
HostTensor(Coord_t const& _stride, Coord_t const& _size) { reset(_stride, _size); }
/// Constructs a HostTensor from size - infers strides
HostTensor(Coord_t const& _size) {
Coord_t _stride = make_Coord(
_size.at(2) * _size.at(1) * _size.at(0), _size.at(1) * _size.at(0), _size.at(0), 1);
reset(_stride, _size);
/// Constructor for resizing the least significant rank
HostTensor(Index_ size_1D, bool device_backed = true) {
this->resize(size_1D, device_backed);
}
/// Returns the number of elements needed to back vector
size_t capacity() { return Base::capacity(); }
/// Helper to construct from pointer, stride, and size
HostTensor(
StorageCoord const &_stride,
TensorCoord const& _size,
bool _device_backed = true
) {
/// Returns true if the Tensor_view is bound to some memory
bool good() const { return Base::good(); }
this->reset(_stride, _size);
}
/// Clears the HostTensor allocation to size/capacity = 0
void reset() {
host_.clear();
device_.reset();
Base::reset();
}
/// Helper to resize the least significant rank
void resize(
Index_ size_1D,
bool _device_backed = true) {
TensorCoord _size;
_size[Base::kRank - 1] = size_1D;
for (int i = 0; i < Base::kRank - 1; ++i) {
_size[i] = 1;
}
StorageCoord _stride;
_stride[Base::kStorageRank - 1] = 1;
for (int i = 0; i < Base::kStorageRank - 1; ++i) {
_stride[i] = size_1D;
}
this->reset(_stride, _size, _device_backed);
}
/// Updates the reference and size of a Tensor_view object
void reset(Coord_t const& _stride, Coord_t const& _size) {
size_t _capacity = _size.at(0) * _stride.at(0);
void reset(
StorageCoord const& stride,
TensorCoord const& size,
bool _device_backed = true) {
// Construct a temporary TensorView so we can calculate the new capacity
size_t _capacity = Base(nullptr, stride, size).capacity();
// Allocate memory
DeviceType* _device_memory = nullptr;
if (DeviceBacked) {
if (_device_backed) {
_device_memory = cutlass::device_memory::allocate<DeviceType>(_capacity);
}
host_.clear();
host_.resize(_capacity);
for (size_t i = 0; i < _capacity; ++i) {
host_[i] = T((int)0xdeadbeef);
}
device_.reset(_device_memory, _capacity);
Base::reset(TensorRef_t(host_.data(), _stride), _size);
Base::reset(TensorRef(host_.data(), stride), size);
}
/// Initializes the host tensor as a matrix
void resize_matrix(int rows, int columns, MatrixLayout::Kind layout) {
bool col_major = (layout == MatrixLayout::kColumnMajor);
int ldm = (col_major ? rows : columns);
/// Accesses the tensor reference pointing to data
TensorRef host_ref() { return Base::ref(); }
Coord_t stride = make_Coord(rows * columns, col_major ? 1 : ldm, col_major ? ldm : 1, 1);
/// Accesses the tensor reference pointing to data
TensorRef host_ref() const { return Base::ref(); }
Coord_t size = make_Coord(1, rows, columns, 1);
reset(stride, size);
/// Accesses the tensor reference pointing to data
DeviceTensorRef device_ref() const {
return DeviceTensorRef(device_data(), this->stride());
}
/// Simplifies resizing the host tensor
void resize(int elements) { resize_matrix(1, elements, MatrixLayout::kColumnMajor); }
/// Accesses the tensor reference pointing to data
HostTensorView host_view() {
return HostTensorView(host_data(), this->stride(), this->size());
}
/// Accesses the tensor reference pointing to data
ConstHostTensorView host_view() const {
return HostTensorView(host_data(), this->stride(), this->size());
}
/// Accesses the tensor reference pointing to data
DeviceTensorView device_view() const {
return DeviceTensorView(device_data(), this->stride(), this->size());
}
/// Gets pointer to host data
T const* host_data() const { return &host_[0]; }
/// Gets pointer to host data
T* host_data() { return &host_[0]; }
HostType * host_data() { return host_.data(); }
/// Gets pointer to device data
DeviceType* device_data() const { return device_.get(); }
DeviceType* device_data() { return device_.get(); }
/// Gets pointer to host data
HostType const * host_data() const { return host_.data(); }
/// Gets pointer to device data
DeviceType * device_data() const { return device_.get(); }
/// Returns true if device memory is allocated
bool device_backed() const {
return device_.get();
}
/// Copies data from device to host
void sync_host() {
if (DeviceBacked) {
if (device_.get()) {
device_memory::copy_to_host(
host_.data(), reinterpret_cast<T const*>(device_.get()), host_.size());
host_.data(), reinterpret_cast<HostType const*>(device_.get()), host_.size());
}
}
/// Copies data from host to device
void sync_device() {
if (DeviceBacked) {
if (device_.get()) {
device_memory::copy_to_device(
device_.get(), reinterpret_cast<DeviceType const*>(host_.data()), host_.size());
device_.get(),
reinterpret_cast<DeviceType const*>(host_.data()),
host_.size());
}
}
/// Copy data from a caller-supplied device pointer
void copy_to_host(DeviceType const *ptr_device) {
/// Copy data from a caller-supplied device pointer into host memory
void copy_to_host(DeviceType const* ptr_device) {
device_memory::copy_to_host(
host_.data(), reinterpret_cast<T const *>(ptr_device), host_.size());
host_.data(), reinterpret_cast<HostType const*>(ptr_device), host_.size());
}
/// Copies data to a caller-supplied device pointer
void copy_to_device(DeviceType *ptr_device) {
/// Copies device-to-device
void copy_to_device(DeviceType* ptr_device) {
device_memory::copy_to_device(
ptr_device, reinterpret_cast<DeviceType const *>(host_.data()), host_.size());
}
/// Accesses the tensor reference pointing to data
TensorRef_t& host_ref() { return Base::ref(); }
/// Accesses the tensor reference pointing to data
TensorRef_t const& host_ref() const { return Base::ref(); }
/// Accesses the tensor reference pointing to data
DeviceTensorRef device_ref() const { return DeviceTensorRef(device_data(), stride()); }
/// Returns a tensor ref to constant memory on the device
ConstDeviceTensorRef const_device_ref() const {
return ConstDeviceTensorRef(device_data(), stride());
}
/// Accesses the size
Coord_t const& size() const { return Base::size(); }
/// Accesses the size
int size(int dim) const { return Base::size(dim); }
/// Accesses the size
Coord_t const& stride() const { return Base::stride(); }
/// Accesses the size
int stride(int dim) const { return Base::stride(dim); }
/// Returns the index of an element
Offset_t offset(Coord_t const& coord) const { return Base::offset(coord); }
/// Determines whether a location is within a tensor
bool contains(Coord_t const& coord) const { return Base::contains(coord); }
/// Element-wise accessor
T& at(Coord_t const& coord) const { return Base::at(coord); }
/// Element-wise accessor
T& operator[](Coord_t const& coord) { return at(coord); }
/// Element-wise accessor with basic offset
T& at(int idx) const { return Base::at(idx); }
/// Returns a Tensor_view given location and size quantities
TensorView<T> subview(Coord_t const& _location, Coord_t _size) const {
return Base::subview(_location, _size);
}
/// Recurses through all dimensions and applies a unary operation
template <typename F>
void elementwise_in_place(F& op, int dim = 0, Offset_t dst_offset_base = 0) {
Base::elementwise_in_place(op, dim, dst_offset_base);
}
/// Recurses through all dimensions and applies a unary operator, supplying the logical
/// coordinate within the tensor as an argument
template <typename F>
void elementwise_stream(F& op, int dim = 0, Offset_t dst_offset_base = 0) {
Base::elementwise_stream(op, dim, dst_offset_base);
}
/// Recurses through all dimensions and applies a unary operator, supplying the logical
/// coordinate within the tensor as an argument
template <typename F>
void elementwise_generate(F& op,
int dim = 0,
Offset_t dst_offset_base = 0,
Coord_t coord = Coord_t(0)) {
Base::elementwise_generate(op, dim, dst_offset_base, coord);
}
/// Recurses through all dimensions and applies a binary operation
template <typename Src, typename F>
bool elementwise_in_place(F& op,
int dim,
TensorView<Src> const& tensor,
Offset_t dst_offset_base = 0,
Offset_t src_offset_base = 0) {
return Base::elementwise_in_place(op, dim, tensor, dst_offset_base, src_offset_base);
ptr_device, reinterpret_cast<DeviceType const*>(host_.data()), host_.size());
}
/// Accumulate in place
template <typename Src>
TensorView<T>& operator+=(TensorView<Src> const& tensor) {
template <typename SrcTensorView>
HostTensor& operator+=(SrcTensorView const& tensor) {
Base::operator+=(tensor);
sync_device();
return *this;
}
/// Subtract in place
template <typename Src>
TensorView<T>& operator-=(TensorView<Src> const& tensor) {
template <typename SrcTensorView>
HostTensor& operator-=(SrcTensorView const& tensor) {
Base::operator-=(tensor);
sync_device();
return *this;
}
/// Multiply in place
template <typename Src>
TensorView<T>& operator*=(TensorView<Src> const& tensor) {
template <typename SrcTensorView>
HostTensor& operator*=(SrcTensorView const& tensor) {
Base::operator*=(tensor);
sync_device();
return *this;
}
/// Divide in place
template <typename Src>
TensorView<T>& operator/=(TensorView<Src> const& tensor) {
template <typename SrcTensorView>
HostTensor& operator/=(SrcTensorView const& tensor) {
Base::operator/=(tensor);
sync_device();
return *this;
}
/// equality with epsilon tolerance
bool equals(TensorView<T> const& tensor, T epsilon) const {
return Base::equals(tensor, epsilon);
}
/// equality with ulps tolerance
bool bit_equals(TensorView<T> const& tensor, long long ulps_threshold = 0) {
return Base::bit_equals(tensor, ulps_threshold);
}
/// Computes general matrix product among select dimensions of a tensor
/// Assumes:
/// D: number of independent GEMMs to compute
/// H: height of matrix
/// W: width of matrix
template <
/// Data type of A matrix elements
typename A,
/// Data type of B matrix elements
typename B,
/// Data type of "compute" type (i.e. accumulator)
typename Ctype,
/// Data type of scale factors
typename Stype>
void gemm(TensorView<A> const& tensor_a, TensorView<B> const& tensor_b, Stype alpha, Stype beta) {
Base::template gemm<A, B, Ctype, Stype>(tensor_a, tensor_b, alpha, beta);
}
/// Fills with random data
template <typename Gen>
void fill_random(Gen generator) {
@@ -335,31 +351,38 @@ class HostTensor : public HostTensorView<T> {
}
/// computes elements as a linear combination of their coordinates
void fill_linear(Coord_t v, T offset = T(0)) {
void fill_linear(TensorCoord v, HostType offset = HostType(0)) {
Base::fill_linear(v, offset);
sync_device();
}
/// computes elements as a linear combination of their coordinates
void fill_sequential(T v = T(1), T offset = T(0)) {
void fill_sequential(HostType v = HostType(1), HostType offset = HostType(0)) {
Base::fill_sequential(v, offset);
sync_device();
}
/// fills with a value
void fill(T val = T(0)) {
void fill(HostType val = HostType(0)) {
Base::fill(val);
sync_device();
}
/// Copies from external data source and performs type conversion
template <typename Src>
void fill(TensorView<Src> const& tensor) {
/// copies from external data source and performs type conversion
template <
typename SrcType,
typename SrcMapFunc_,
int SrcStorageRank_,
typename SrcIndex_,
typename SrcLongIndex_
>
void fill(
TensorView<SrcType, Base::kRank, SrcMapFunc_, SrcStorageRank_, SrcIndex_, SrcLongIndex_> const& tensor) {
Base::fill(tensor);
sync_device();
}
/// Computes the norm of the matrix in double-precision
double norm() const { return Base::norm(); }
};
///////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace cutlass
tools/util/host_tensor_view.h
+199 -246
View File
@@ -23,45 +23,77 @@
*
**************************************************************************************************/
/*! \file
\brief Host-side implementation of useful operations
\brief Host-side implementation of basic tensor operations.
See cutlass/tensor_ref.h and cutlass/tensor_view.h for more details.
*/
#pragma once
#include <cutlass/cutlass.h>
#include <cutlass/tensor_view.h>
#include <tools/util/type_traits.h>
#include "cutlass/cutlass.h"
#include "cutlass/tensor_view.h"
#include "tools/util/type_traits.h"
namespace cutlass {
////////////////////////////////////////////////////////////////////////////////////////////////////
template <typename SrcType, typename DstType>
struct Cast {
static inline DstType apply(SrcType src) { return static_cast<DstType>(src); };
};
template <>
struct Cast<float, int8_t> {
static inline int8_t apply(float src) {
return static_cast<int8_t>(fmaxf(-128.f, fminf(127.f, src)));
};
};
template <>
struct Cast<float, uint8_t> {
static inline uint8_t apply(float src) {
return static_cast<uint8_t>(fmaxf(0.f, fminf(255.f, src)));
};
};
////////////////////////////////////////////////////////////////////////////////////////////////////
template <typename T>
class HostTensorView : public TensorView<T> {
template <
/// Data type of element stored within tensor
typename Storage_,
/// Rank of logical tensor
int Rank_ = 4,
/// Maps a Coord<Rank_> in the logical tensor index space to the internal n-D array
typename MapFunc_ = IdentityTensorMapFunc<Rank_>,
/// Rank of internal n-D array
int StorageRank_ = Rank_,
/// Index type used for coordinates
typename Index_ = int,
/// Index type used for offsets and pointer differences
typename LongIndex_ = long long
>
class HostTensorView :
public TensorView<Storage_, Rank_, MapFunc_, StorageRank_, Index_, LongIndex_> {
public:
/// Base class
typedef TensorView<T> TensorView_t;
typedef TensorView<Storage_, Rank_, MapFunc_, StorageRank_, Index_, LongIndex_> Base;
/// Storage type
typedef typename Base::Storage Storage;
/// Alias for underlying TensorRef
typedef typename Base::TensorRef TensorRef;
/// Index type
typedef typename Base::Index Index;
/// Coordinate in logical tensor space
typedef typename TensorRef::TensorCoord TensorCoord;
/// Coordinate in storage n-D array
typedef typename TensorRef::StorageCoord StorageCoord;
/// Stride vector in storage coordinate space
/// Least significant stride is = 1 and not stored
typedef typename TensorRef::StrideVector StrideVector;
/// Long index type for pointer offsets
typedef typename Base::LongIndex LongIndex;
/// Rank of tensor index space
static int const kRank = Base::kRank;
//
// Definitions included for backwards compatibility - These will be remmoved
// in the next major release.
//
/// Base class
typedef Base TensorView_t;
//
// These definitions are meaningful for rank=4 tensors.
//
/// Convention: depth is the first dimension
static int const Dim_D = 0;
@@ -75,19 +107,8 @@ class HostTensorView : public TensorView<T> {
/// Convention: channel is the second dimension
static int const Dim_C = 3;
/// Rank of tensor
static int const Rank = TensorView_t::Rank;
/// Type used to compute the offset of an element to the base of a tensor
typedef typename TensorView_t::Offset_t Offset_t;
/// Reference and stride
typedef typename TensorView_t::TensorRef_t TensorRef_t;
/// Coordinate into tensor
typedef typename TensorView_t::Coord_t Coord_t;
public:
//
// Device and Host Methods
//
@@ -95,91 +116,87 @@ class HostTensorView : public TensorView<T> {
/// Default constructor
HostTensorView() {}
/// Constructs a Tensor_view from a TensorRef and size
HostTensorView(TensorRef_t const& _ref, Coord_t const& _size) : TensorView_t(_ref, _size) {}
/// Helper to construct from pointer, stride, and size
HostTensorView(
Storage_ *_ptr,
StrideVector const &_stride,
TensorCoord const& _size
) : Base(TensorRef(_ptr, _stride), _size) {}
/// Accesses the size
Coord_t const& size() const { return TensorView_t::size(); }
/// Helper to construct from pointer, stride, and size
HostTensorView(
Storage_ *_ptr,
StorageCoord const &_stride,
TensorCoord const& _size
) : Base(TensorRef(_ptr, _stride), _size) {}
/// Accesses the size of a specified dimension
int size(int dim) const { return size().at(dim); }
/// Accesses the stride
Coord_t const& stride() const { return TensorView_t::stride(); }
/// Accesses the stride along a specified dimension
int stride(int dim) const { return stride().at(dim); }
/// Returns the number of scalar elements needed to store tensor
size_t capacity() const { return size(3) * stride(3) * stride(2) * stride(1) * stride(0); }
/// Returns true if the Tensor_view is bound to some memory
bool good() const { return TensorView_t::good(); }
/// Updates the reference and size of a TensorView object
void reset(TensorRef_t const& _ref = TensorRef_t(0), Coord_t const& _size = Coord_t()) {
return TensorView_t::reset(_ref, _size);
}
/// Accesses the tensor reference pointing to data
TensorRef_t& ref() { return TensorView_t::ref(); }
/// Accesses the tensor reference pointing to data
TensorRef_t const& ref() const { return TensorView_t::ref(); }
/// Constructs a Tensor_view from a TensorRef and size assuming dense packing
HostTensorView(
TensorRef const& _ref,
TensorCoord const& _size) : Base(_ref, _size) {}
/// Assigns a tensor view
HostTensorView& operator=(TensorView_t const& _tensor) {
reset(_tensor.ref(), _tensor.size());
HostTensorView& operator=(Base const& _tensor) {
this->reset(_tensor.ref(), _tensor.size());
return *this;
}
/// Returns the index of an element
Offset_t offset(Coord_t const& coord) const { return TensorView_t::offset(coord); }
/// Returns a TensorView offset by a given amount
CUTLASS_HOST_DEVICE
HostTensorView operator+(TensorCoord const& b) const {
HostTensorView result(*this);
result.add_pointer_offset(this->offset(b));
return result;
}
/// Determines whether a location is within a tensor
bool contains(Coord_t const& coord) const { return TensorView_t::contains(coord); }
/// Returns a TensorRef offset by a given amount
CUTLASS_HOST_DEVICE
HostTensorView& operator+=(TensorCoord const& b) {
this->add_pointer_offset(this->offset(b));
return *this;
}
/// Element-wise accessor
T& at(Coord_t const& coord) const { return TensorView_t::at(coord); }
/// Returns a TensorRef offset by a given amount
CUTLASS_HOST_DEVICE
HostTensorView operator-(TensorCoord const& b) const {
TensorRef result(*this);
result.add_pointer_offset(-this->offset(b));
return result;
}
/// Element-wise accessor
T& operator[](Coord_t const& coord) const { return at(coord); }
/// Accesses an element with a raw offset
T& at(int idx) const { return TensorView_t::at(idx); }
/// Accesses an element with a raw offset
T& operator[](int idx) const { return at(idx); }
/// Returns a Tensor_view given location and size quantities
TensorView_t subview(Coord_t const& location, Coord_t size) const {
return TensorView_t::subview(location, size);
/// Returns a TensorRef offset by a given amount
CUTLASS_HOST_DEVICE
HostTensorView& operator-=(TensorCoord const& b) {
this->add_pointer_offset(-this->offset(b));
return *this;
}
/// Recurses through all dimensions and applies a unary operation in place
template <typename F>
void elementwise_in_place(F& op, int dim = 0, Offset_t dst_offset_base = 0) {
Offset_t dst_offset = dst_offset_base;
void elementwise_in_place(F& op, int dim = 0, TensorCoord const &start_coord = TensorCoord()) {
for (int idx = 0; idx < size(dim); ++idx, dst_offset += stride(dim)) {
if (dim < Rank - 1) {
elementwise_in_place(op, dim + 1, dst_offset);
TensorCoord coord(start_coord);
for (int idx = 0; idx < this->size(dim); ++idx) {
coord[dim] = idx;
if (dim < kRank - 1) {
elementwise_in_place(op, dim + 1, coord);
} else {
op(ref().data()[dst_offset]);
op(this->at(coord));
}
}
}
/// Recurses through all dimensions and applies a unary operator with no arguments
template <typename F>
void elementwise_stream(F& op, int dim = 0, Offset_t dst_offset_base = 0) {
Offset_t dst_offset = dst_offset_base;
void elementwise_stream(F& op, int dim = 0, TensorCoord const &start_coord = TensorCoord()) {
for (int idx = 0; idx < size(dim); ++idx, dst_offset += stride(dim)) {
if (dim < Rank - 1) {
elementwise_stream(op, dim + 1, dst_offset);
TensorCoord coord(start_coord);
for (int idx = 0; idx < this->size(dim); ++idx) {
coord[dim] = idx;
if (dim < kRank - 1) {
elementwise_stream(op, dim + 1, coord);
} else {
ref().data()[dst_offset] = op();
this->at(coord) = op();
}
}
}
@@ -189,61 +206,56 @@ class HostTensorView : public TensorView<T> {
template <typename F>
void elementwise_generate(F& op,
int dim = 0,
Offset_t dst_offset_base = 0,
Coord_t coord = Coord_t(0)) {
Offset_t dst_offset = dst_offset_base;
TensorCoord const & start_coord = TensorCoord()) {
for (int idx = 0; idx < size(dim); ++idx, dst_offset += stride(dim)) {
coord.at(dim) = idx;
if (dim < Rank - 1) {
elementwise_generate(op, dim + 1, dst_offset, coord);
TensorCoord coord(start_coord);
for (int idx = 0; idx < this->size(dim); ++idx) {
coord[dim] = idx;
if (dim < kRank - 1) {
elementwise_generate(op, dim + 1, coord);
} else {
ref().data()[dst_offset] = op(coord);
this->at(coord) = op(coord);
}
}
}
/// Recurses through all dimensions and applies a unary operator, supplying the logical
/// coordinate within the tensor as an argument
/// coordinate within the tensor as an argument. Mutable.
template <typename F>
void elementwise_visit(F& op,
int dim = 0,
Offset_t dst_offset_base = 0,
Coord_t coord = Coord_t(0)) const {
Offset_t dst_offset = dst_offset_base;
TensorCoord const & start_coord = TensorCoord()) const {
for (int idx = 0; idx < size(dim); ++idx, dst_offset += stride(dim)) {
coord.at(dim) = idx;
TensorCoord coord(start_coord);
for (int idx = 0; idx < this->size(dim); ++idx) {
coord[dim] = idx;
if (dim < Rank - 1) {
elementwise_visit(op, dim + 1, dst_offset, coord);
if (dim < kRank - 1) {
elementwise_visit(op, dim + 1, coord);
} else {
op(ref().data()[dst_offset], coord);
op(this->at(coord), coord);
}
}
}
/// Recurses through all dimensions and applies a binary operation
template <typename Src, typename F>
template <typename F, typename SrcTensorView>
bool elementwise_in_place(F& op,
TensorView<Src> const& tensor,
SrcTensorView const& tensor,
int dim = 0,
Offset_t dst_offset_base = 0,
Offset_t src_offset_base = 0) {
Offset_t dst_offset = dst_offset_base;
Offset_t src_offset = src_offset_base;
TensorCoord const &start_coord = TensorCoord()) {
if (size().at(dim) != tensor.size().at(dim)) {
if (this->size(dim) != tensor.size(dim)) {
return false;
}
for (int idx = 0; idx < size(dim);
++idx, dst_offset += stride(dim), src_offset += tensor.stride(dim)) {
if (dim < Rank - 1) {
elementwise_in_place(op, tensor, dim + 1, dst_offset, src_offset);
TensorCoord coord(start_coord);
for (int idx = 0; idx < this->size(dim); ++idx) {
coord[dim] = idx;
if (dim < kRank - 1) {
elementwise_in_place(op, tensor, dim + 1, coord);
} else {
op(data()[dst_offset], tensor.data()[src_offset]);
op(this->at(coord), tensor.at(coord));
}
}
@@ -252,55 +264,55 @@ class HostTensorView : public TensorView<T> {
template <typename Src>
struct LambdaBinaryAddition {
void operator()(T& a, Src b) const { a += T(b); }
void operator()(Storage_& a, Src b) const { a += Storage_(b); }
};
template <typename Src>
struct LambdaBinarySubtraction {
void operator()(T& a, Src b) const { a -= T(b); }
void operator()(Storage_& a, Src b) const { a -= Storage_(b); }
};
template <typename Src>
struct LambdaBinaryMultiplication {
void operator()(T& a, Src b) const { a *= T(b); }
void operator()(Storage_& a, Src b) const { a *= Storage_(b); }
};
template <typename Src>
struct LambdaBinaryDivision {
void operator()(T& a, Src b) const { a /= T(b); }
void operator()(Storage_& a, Src b) const { a /= Storage_(b); }
};
/// Accumulate in place
template <typename Src>
TensorView<T>& operator+=(TensorView<Src> const& tensor) {
LambdaBinaryAddition<Src> op;
template <typename SrcTensorView>
HostTensorView& operator+=(SrcTensorView const& tensor) {
LambdaBinaryAddition<typename SrcTensorView::Storage> op;
elementwise_in_place(op, tensor);
return *this;
}
/// Subtract in place
template <typename Src>
TensorView<T>& operator-=(TensorView<Src> const& tensor) {
LambdaBinarySubtraction<Src> op;
template <typename SrcTensorView>
HostTensorView& operator-=(SrcTensorView const& tensor) {
LambdaBinarySubtraction<typename SrcTensorView::Storage> op;
elementwise_in_place(op, tensor);
return *this;
}
/// Multiply in place
template <typename Src>
TensorView<T>& operator*=(TensorView<Src> const& tensor) {
LambdaBinaryMultiplication<Src> op;
template <typename SrcTensorView>
HostTensorView& operator*=(SrcTensorView const& tensor) {
LambdaBinaryMultiplication<typename SrcTensorView::Storage> op;
elementwise_in_place(op, tensor);
return *this;
}
/// Divide in place
template <typename Src>
TensorView<T>& operator/=(TensorView<Src> const& tensor) {
LambdaBinaryDivision<Src> op;
template <typename SrcTensorView>
HostTensorView& operator/=(SrcTensorView const& tensor) {
LambdaBinaryDivision<typename SrcTensorView::Storage> op;
elementwise_in_place(op, tensor);
return *this;
@@ -309,19 +321,19 @@ class HostTensorView : public TensorView<T> {
/// Comparison operator
struct EqualsOperator {
bool equal;
T eps;
Storage_ eps;
EqualsOperator(T _epsilon) : equal(true), eps(_epsilon) {}
EqualsOperator(Storage_ _epsilon) : equal(true), eps(_epsilon) {}
void operator()(T a, T b) {
if (std::abs(T(a - b)) > eps * std::max(std::abs(a), std::abs(b))) {
void operator()(Storage_ a, Storage_ b) {
if (std::abs(Storage_(a - b)) > eps * std::max(std::abs(a), std::abs(b))) {
equal = false;
}
}
};
/// equality with epsilon tolerance
bool equals(TensorView<T> const& tensor, T epsilon) const {
bool equals(Base const& tensor, Storage epsilon) const {
EqualsOperator comparison_op(epsilon);
bool equal_size = elementwise_in_place(comparison_op, tensor);
@@ -336,13 +348,13 @@ class HostTensorView : public TensorView<T> {
BitEqualsOperator(long long _ulps_threshold) : equal(true), eps(_ulps_threshold), index(0) {}
void operator()(T a, T b) {
void operator()(Storage_ a, Storage_ b) {
// convert bits to integers
long long bits_a = 0;
long long bits_b = 0;
*reinterpret_cast<T*>(&bits_a) = TypeTraits<T>::remove_negative_zero(a);
*reinterpret_cast<T*>(&bits_b) = TypeTraits<T>::remove_negative_zero(b);
*reinterpret_cast<Storage_*>(&bits_a) = TypeTraits<Storage_>::remove_negative_zero(a);
*reinterpret_cast<Storage_*>(&bits_b) = TypeTraits<Storage_>::remove_negative_zero(b);
// compute diff
long long ulps = bits_a - bits_b;
@@ -354,85 +366,13 @@ class HostTensorView : public TensorView<T> {
};
/// equality with ulps tolerance
bool bit_equals(TensorView<T> const& tensor, long long ulps_threshold = 0) {
bool bit_equals(Base const& tensor, long long ulps_threshold = 0) {
BitEqualsOperator comparison_op(ulps_threshold);
bool equal_size = elementwise_in_place(comparison_op, tensor);
return equal_size && comparison_op.equal;
}
/// Gets naked pointer to data
T* data() const { return TensorView_t::data(); }
/// Computes general matrix product among select dimensions of a tensor
/// Assumes:
/// D: number of independent GEMMs to compute
/// H: height of matrix
/// W: width of matrix
/// C: "channels" of each element
template <typename A, typename B, typename Ctype, typename Stype>
void gemm(TensorView<A> const& tensor_a, TensorView<B> const& tensor_b, Stype alpha, Stype beta) {
int const Batch = size(Dim_D);
int const M = size(Dim_H);
int const N = size(Dim_W);
int const K = tensor_a.size(Dim_W);
int const C = tensor_a.size(Dim_C);
// Sizes must match
if (tensor_a.size(Dim_H) != M || tensor_b.size(Dim_W) != N || tensor_b.size(Dim_C) != C ||
tensor_b.size(Dim_H) != K) {
return;
}
int const Mblock = 32;
int const Nblock = 32;
for (int batch = 0; batch < Batch; ++batch) {
for (int row_block = 0; row_block < M; row_block += Mblock) {
for (int col_block = 0; col_block < N; col_block += Nblock) {
Ctype accum[Mblock][Nblock];
for (int j = 0; j < Nblock; j++) {
for (int i = 0; i < Mblock; i++) {
accum[i][j] = Ctype(0);
}
}
for (int k_block = 0; k_block < K; ++k_block) {
for (int j = 0; j < Nblock; j++) {
for (int i = 0; i < Mblock; i++) {
int row = row_block + i;
int col = col_block + j;
if (row < M && col < N) {
for (int channel = 0; channel < C; ++channel) {
Ctype a(tensor_a.at(make_Coord(batch, row, k_block, channel)));
Ctype b(tensor_b.at(make_Coord(batch, k_block, col, channel)));
accum[i][j] += a * b;
}
}
}
}
}
for (int j = 0; j < Nblock; j++) {
for (int i = 0; i < Mblock; i++) {
int row = row_block + i;
int col = col_block + j;
Coord_t coord = make_Coord(batch, row, col, 0);
if (row < M && col < N) {
at(coord) =
Cast<Stype, T>::apply(alpha * Stype(accum[i][j]) + beta * Stype(at(coord)));
}
}
}
}
}
}
}
/// Fills with random data
template <typename Gen>
void fill_random(Gen generator) {
@@ -453,7 +393,9 @@ class HostTensorView : public TensorView<T> {
/// Generator to fill a tensor with the identity matrix
struct LambdaFillIdentity {
T operator()(Coord_t const& coord) { return (coord.at(1) == coord.at(2) ? T(1) : T(0)); }
Storage_ operator()(TensorCoord const& coord) {
return (coord.at(1) == coord.at(2) ? Storage_(1) : Storage_(0));
}
};
/// initializes with identity
@@ -464,39 +406,41 @@ class HostTensorView : public TensorView<T> {
/// Lambda for fill_linear()
struct LambdaFillLinear {
Coord_t v_;
T offset_;
TensorCoord v_;
Storage_ offset_;
LambdaFillLinear(Coord_t const& _v, T _offset) : v_(_v), offset_(_offset) {}
LambdaFillLinear(TensorCoord const& _v, Storage_ _offset) : v_(_v), offset_(_offset) {}
T operator()(Coord_t const& coord) { return T(v_.template dot<int>(coord)) + offset_; }
Storage_ operator()(TensorCoord const& coord) {
return Storage_(v_.template dot<int>(coord)) + offset_;
}
};
/// computes elements as a linear combination of their coordinates
void fill_linear(Coord_t v, T offset = T(0)) {
void fill_linear(TensorCoord v, Storage_ offset = Storage_(0)) {
LambdaFillLinear lambda(v, offset);
elementwise_generate(lambda);
}
/// computes elements as a linear combination of their coordinates
void fill_sequential(T v = T(1), T offset = T(0)) {
int const count = size().count();
void fill_sequential(Storage_ v = Storage_(1), Storage_ offset = Storage_(0)) {
int const count = this->size().count();
for (int i = 0; i < count; ++i) {
data()[i] = T(i);
this->data()[i] = Storage_(i);
}
}
/// Returns a constant value
struct LambdaFillValue {
T value;
Storage_ value;
LambdaFillValue(T _value) : value(_value) {}
LambdaFillValue(Storage_ _value) : value(_value) {}
T operator()() { return value; }
Storage_ operator()() { return value; }
};
/// fills with a value
void fill(T val = T(0)) {
void fill(Storage_ val = Storage_(0)) {
LambdaFillValue op(val);
elementwise_stream(op);
}
@@ -504,13 +448,21 @@ class HostTensorView : public TensorView<T> {
/// Conversion from Src to T
template <typename Src>
struct LambdaAssign {
void operator()(T& a, Src b) const { a = T(b); }
void operator()(Storage_& a, Src b) const { a = Storage_(b); }
};
/// copies from external data source and performs type conversion
template <typename Src>
void fill(TensorView<Src> const& tensor) {
LambdaAssign<Src> op;
template <
typename SrcType,
typename SrcMapFunc_,
int SrcStorageRank_,
typename SrcIndex_,
typename SrcLongIndex_
>
void fill(
TensorView<SrcType, kRank, SrcMapFunc_, SrcStorageRank_, SrcIndex_, SrcLongIndex_> const& tensor) {
LambdaAssign<SrcType> op;
elementwise_in_place(op, tensor);
}
@@ -520,7 +472,7 @@ class HostTensorView : public TensorView<T> {
LambdaNorm() : sum(0) {}
void operator()(T const& element) {
void operator()(Storage const& element) {
double value(element);
double conj(element); // TODO - conjugates for complex
@@ -540,3 +492,4 @@ class HostTensorView : public TensorView<T> {
////////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace cutlass
tools/util/reference/device/kernel/tensor_elementwise.h
+162
View File
@@ -0,0 +1,162 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#include <curand_kernel.h>
#include "cutlass/cutlass.h"
namespace cutlass {
namespace reference {
namespace device {
namespace kernel {
////////////////////////////////////////////////////////////////////////////////////////////////////
/// Kernel to initialize tensor to uniform random distribution
template <typename T>
__global__ void TensorInitializeUniform(
Distribution dist, int64_t seed, int dim_contiguous, int dim_strided, T *tensor, int ldm) {
__shared__ curandState_t rng_state[1024];
uint64_t gtid = threadIdx.x + blockIdx.x * blockDim.x + blockIdx.y * gridDim.x * blockDim.x;
curand_init(seed, gtid, 0, &rng_state[threadIdx.x]);
int c_idx = blockIdx.x * blockDim.x + threadIdx.x;
int s_idx = blockIdx.y * blockDim.x;
tensor += s_idx * ldm + c_idx;
for (int s_offset = 0; s_offset < blockDim.x; ++s_offset, ++s_idx) {
if (s_idx < dim_strided && c_idx < dim_contiguous) {
double range = dist.uniform.max - dist.uniform.min;
double rnd = curand_uniform(&rng_state[threadIdx.x]);
rnd = dist.uniform.min + range * rnd;
// Random values are cast to integer after scaling by a power of two to facilitate error
// testing
if (dist.int_scale >= 0) {
rnd = double(int(rnd * double(1 << dist.int_scale)));
*tensor = T(rnd / double(1 << dist.int_scale));
} else {
*tensor = T(rnd);
}
tensor += ldm;
}
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Kernel to initialize tensor to uniform distribution
template <typename T>
__global__ void TensorInitializeGaussian(
Distribution dist, int64_t seed, int dim_contiguous, int dim_strided, T *tensor, int ldm) {
__shared__ curandState_t rng_state[1024];
uint64_t gtid = threadIdx.x + blockIdx.x * blockDim.x + blockIdx.y * gridDim.x * blockDim.x;
curand_init(seed, gtid, 0, &rng_state[threadIdx.x]);
int c_idx = blockIdx.x * blockDim.x + threadIdx.x;
int s_idx = blockIdx.y * blockDim.x;
tensor += s_idx * ldm + c_idx;
for (int s_offset = 0; s_offset < blockDim.x; ++s_offset, ++s_idx) {
if (s_idx < dim_strided && c_idx < dim_contiguous) {
// Random values are cast to integer after scaling by a power of two to facilitate error
// testing
double rnd = curand_normal(&rng_state[threadIdx.x]);
rnd = dist.gaussian.mean + dist.gaussian.stddev * rnd;
if (dist.int_scale >= 0) {
rnd = double(int(rnd * double(1 << dist.int_scale)));
*tensor = T(rnd / double(1 << dist.int_scale));
} else {
*tensor = T(rnd);
}
}
}
}
/// Kernel to initialize tensor to an identity matrix
template <typename T>
__global__ void TensorInitializeLinear(
Distribution dist, int64_t seed, int dim_contiguous, int dim_strided, T *tensor, int ldm) {
__shared__ curandState_t rng_state[1024];
uint64_t gtid = threadIdx.x + blockIdx.x * blockDim.x + blockIdx.y * gridDim.x * blockDim.x;
curand_init(seed, gtid, 0, &rng_state[threadIdx.x]);
int c_idx = blockIdx.x * blockDim.x + threadIdx.x;
int s_idx = blockIdx.y * blockDim.x;
tensor += s_idx * ldm + c_idx;
for (int s_offset = 0; s_offset < blockDim.x; ++s_offset, ++s_idx) {
if (s_idx < dim_strided && c_idx < dim_contiguous) {
*tensor =
dist.linear.offset + dist.linear.delta_row * c_idx + dist.linear.delta_column * s_idx;
}
}
}
/// Kernel to initialize tensor to an identity matrix
template <typename T>
__global__ void TensorInitializeIdentity(
Distribution dist, int64_t seed, int dim_contiguous, int dim_strided, T *tensor, int ldm) {
__shared__ curandState_t rng_state[1024];
uint64_t gtid = threadIdx.x + blockIdx.x * blockDim.x + blockIdx.y * gridDim.x * blockDim.x;
curand_init(seed, gtid, 0, &rng_state[threadIdx.x]);
int c_idx = blockIdx.x * blockDim.x + threadIdx.x;
int s_idx = blockIdx.y * blockDim.x;
tensor += s_idx * ldm + c_idx;
for (int s_offset = 0; s_offset < blockDim.x; ++s_offset, ++s_idx) {
if (s_idx < dim_strided && c_idx < dim_contiguous) {
*tensor = (c_idx == s_idx ? T(1) : T(0));
}
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace kernel
} // namespace device
} // namespace reference
} // namespace cutlass
tools/util/reference/device/kernel/tensor_foreach.h
+112
View File
@@ -0,0 +1,112 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#include "cutlass/cutlass.h"
#include "cutlass/coord.h"
namespace cutlass {
namespace reference {
namespace device {
namespace kernel {
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Defines several helpers
namespace detail {
/// Helper to perform for-each operation
template <typename Func, int Rank, int RankRemaining>
struct TensorForEachHelper {
/// Constructor for general rank
__inline__ __device__
TensorForEachHelper(Func &func, Coord<Rank> const &size, Coord<Rank> &coord, int64_t index) {
int64_t product = 1;
CUTLASS_PRAGMA_UNROLL
for (int i = Rank - RankRemaining; i < Rank; ++i) {
product *= size[i];
}
coord[Rank - 1 - RankRemaining] = index / product;
int64_t remaining = index % product;
TensorForEachHelper<Func, Rank, RankRemaining-1>(func, size, coord, remaining);
}
};
/// Helper to perform for-each operation
template <typename Func, int Rank>
struct TensorForEachHelper<Func, Rank, 0> {
/// Constructor for fastest chaning rank
__inline__ __device__
TensorForEachHelper(Func &func, Coord<Rank> const &size, Coord<Rank> &coord, int64_t index) {
coord[Rank - 1] = index;
if (coord < size) {
func(coord);
}
}
};
} // namespace detail
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Helper to perform for-each operation
template <typename Func, int Rank, typename Params>
__global__ void TensorForEach(Coord<Rank> size, Params params = Params()) {
Func func(params);
int64_t index = threadIdx.x + blockIdx.x * blockDim.x;
int64_t max_index = 1;
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < Rank; ++i) {
max_index *= size[i];
}
CUTLASS_PRAGMA_NO_UNROLL
while (index < max_index) {
Coord<Rank> coord;
detail::TensorForEachHelper<Func, Rank, Rank - 1>(func, size, coord, index);
index += blockDim.x * gridDim.x;
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace kernel
} // namespace device
} // namespace reference
} // namespace cutlass
tools/util/reference/device/tensor_elementwise.h
+772
View File
@@ -0,0 +1,772 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Defines device-side elementwise operations on TensorView. Note, the operations defined
in this header are not specialized for any particular data layout and are therefore not
intended to offer the best possible performance. Rather, they are intended to be generic
reference implementations to support the CUTLASS unit tests.
*/
#pragma once
// Standard Library includes
#include <fstream>
#include <ostream>
#include <stdexcept>
#include <string>
#include <utility>
// CUDA includes
#include <cublas_v2.h>
#include <curand_kernel.h>
// Cutlass includes
#include "cutlass/cutlass.h"
#include "tools/util/device_memory.h"
#include "tools/util/distribution.h"
#include "tools/util/type_traits.h"
#include "tools/util/host_tensor.h"
#include "tools/util/reference/device/tensor_foreach.h"
namespace cutlass {
///////////////////////////////////////////////////////////////////////////////////////////////////
namespace reference {
namespace device {
///////////////////////////////////////////////////////////////////////////////////////////////////
namespace detail {
/// Computes a random uniform distribution
template <typename View_>
struct RandomUniformFunc {
/// View type
typedef View_ View;
/// Scalar type
typedef typename View::Storage T;
/// Coordinate in tensor's index space
typedef typename View::TensorCoord TensorCoord;
/// Parameters structure
struct Params {
/// View object
View view;
/// RNG seed
int64_t seed;
/// Distriubtion
Distribution dist;
/// Default ctor
CUTLASS_HOST_DEVICE
Params() { }
/// Constructor
CUTLASS_HOST_DEVICE
Params(
View const &view,
int64_t seed,
Distribution dist
): view(view), seed(seed), dist(dist) { }
};
//
// Data members
//
/// Parameters object
Params params;
/// RNG state object
curandState_t rng_state;
//
// Methods
//
/// Device-side initialization of RNG
CUTLASS_DEVICE
RandomUniformFunc(Params const &params): params(params) {
uint64_t gtid = threadIdx.x + blockIdx.x * blockDim.x;
curand_init(params.seed, gtid, 0, &rng_state);
}
/// Compute random value and update RNG state
CUTLASS_DEVICE
void operator()(TensorCoord const &coord) {
double range = params.dist.uniform.max - params.dist.uniform.min;
double rnd = curand_uniform(&rng_state);
rnd = params.dist.uniform.min + range * rnd;
// Random values are cast to integer after scaling by a power of two to facilitate error
// testing
T result;
if (params.dist.int_scale >= 0) {
rnd = double(int(rnd * double(1 << params.dist.int_scale)));
result = T(rnd / double(1 << params.dist.int_scale));
}
else {
result = T(rnd);
}
params.view.at(coord) = result;
}
};
/// Computes a random Gaussian distribution
template <typename View_>
struct RandomGaussianFunc {
/// View type
typedef View_ View;
/// Scalar type
typedef typename View::Storage T;
/// Coordinate in tensor's index space
typedef typename View::TensorCoord TensorCoord;
/// Parameters structure
struct Params {
/// View object
View view;
/// RNG seed
int64_t seed;
/// RNG distribution
Distribution dist;
/// Default ctor
CUTLASS_HOST_DEVICE
Params() { }
/// Constructor
CUTLASS_HOST_DEVICE
Params(
View const &view,
int64_t seed,
Distribution dist
): view(view), seed(seed), dist(dist) { }
};
//
// Data members
//
/// Parameters object
Params params;
/// RNG state object
curandState_t rng_state;
//
// Methods
//
/// Device-side initialization of RNG
CUTLASS_DEVICE
RandomGaussianFunc(Params const &params): params(params) {
uint64_t gtid = threadIdx.x + blockIdx.x * blockDim.x;
curand_init(params.seed, gtid, 0, &rng_state);
}
/// Compute random value and update RNG state
CUTLASS_DEVICE
void operator()(TensorCoord const &coord) {
double rnd = curand_normal(&rng_state);
rnd = params.dist.gaussian.mean + params.dist.gaussian.stddev * rnd;
T result;
if (params.dist.int_scale >= 0) {
rnd = double(int(rnd * double(1 << params.dist.int_scale)));
result = T(rnd / double(1 << params.dist.int_scale));
}
else {
result = T(rnd);
}
params.view.at(coord) = result;
}
};
/// Computes a linear combination of each element
template <typename View_>
struct LinearCombinationFunc {
/// View type
typedef View_ View;
/// Scalar type
typedef typename View::Storage T;
/// Coordinate in tensor's index space
typedef typename View::TensorCoord TensorCoord;
//
// Data members
//
/// TensorView object
View view;
/// Delta
Coord<View::kRank, double> delta;
/// Offset
double offset;
//
// Methods
//
/// Constructor
CUTLASS_HOST_DEVICE
LinearCombinationFunc(
View const &view,
Distribution dist
): view(view) {
offset = dist.linear.offset;
if (View::kRank >= 1) {
delta[View::kRank - 1] = dist.linear.delta_column;
}
if (View::kRank >= 2) {
delta[View::kRank - 2] = dist.linear.delta_row;
}
// Additional ranks have delta of zero
for (int i = View::kRank - 2; i > 0; --i) {
delta[i - 1] = 0;
}
}
/// Compute linear combination
CUTLASS_HOST_DEVICE
void operator()(TensorCoord const &coord) {
double result = offset;
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < View::kRank; ++i) {
result += delta[i] * double(coord[i]);
}
view.at(coord) = T(result);
}
};
/// Returns 1 or 0 if the coordinate is along the tensor's diagonal
template <typename View_>
struct IdentityFunc {
/// TensorView
typedef View_ View;
/// Scalar type
typedef typename View::Storage T;
/// Coordinate in tensor's index space
typedef typename View::TensorCoord TensorCoord;
//
// Data members
//
/// View object
View view;
/// Default ctor
CUTLASS_HOST_DEVICE
IdentityFunc(View const &view): view(view) { }
CUTLASS_HOST_DEVICE
void operator()(TensorCoord const &coord) {
bool equal = true;
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < View::kRank; ++i) {
if (coord[i] != coord[0]) {
equal = false;
}
}
view.at(coord) = equal ? T(1) : T(0);
}
};
} // namespace detail
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Initializes a tensor randomly or procedurally.
template <typename View>
void TensorInitialize(View const &view,
int64_t seed,
Distribution const &dist) {
typedef typename View::Storage Scalar;
switch (dist.kind) {
case Distribution::Uniform:
{
typedef detail::RandomUniformFunc<View> Func;
typedef typename Func::Params Params;
TensorForEach<Func, View::kRank, Params>(
view.size(),
Params(view, seed, dist)
);
}
break;
case Distribution::Gaussian:
{
typedef detail::RandomGaussianFunc<View> Func;
typedef typename Func::Params Params;
TensorForEach<Func, View::kRank, Params>(
view.size(),
Params(view, seed, dist)
);
}
break;
case Distribution::Linear:
{
typedef detail::LinearCombinationFunc<View> Func;
TensorForEach<Func, View::kRank, Func>(
view.size(),
Func(view, dist));
}
break;
case Distribution::Identity:
{
typedef detail::IdentityFunc<View> Func;
Func func(view);
TensorForEach<Func, View::kRank, Func>(view.size(), func);
}
break;
default:
break;
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace device
} // namespace reference
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Dispatcher to appropriate initialization kernel - preserved for backwards compatibility
template <typename T>
inline void tensor_initialize(Distribution const &dist,
int64_t seed,
int dim_contiguous,
int dim_strided,
T *tensor,
int ldm) {
TensorView<T, 2> view(tensor, make_Coord(ldm, 1), make_Coord(dim_strided, dim_contiguous));
reference::device::TensorInitialize(view, seed, dist);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
namespace reference {
namespace device {
namespace detail {
/// Compares two tensor views of equal rank and dimension.
template <typename ViewL, typename ViewR>
struct TensorEqualsFunc {
/// Storage type
typedef typename ViewL::Storage T;
/// Unsigned integer type of same size as View type
typedef typename cutlass::TypeTraits<T>::unsigned_type UnsignedType;
/// Coordinate in tensor's index space
typedef typename ViewL::TensorCoord TensorCoord;
/// Assertions
static_assert(ViewL::kRank == ViewR::kRank,
"Cannot compare tensors of different rank");
//
// Data members
//
/// View of left-hand-side tensor
ViewL lhs;
/// View of right-hand-side tensor
ViewR rhs;
/// Pointer to result scalar - only written with 0 if values are incorrect
int *result;
//
// Methods
//
/// Constructor
CUTLASS_HOST_DEVICE
TensorEqualsFunc(ViewL const &lhs, ViewR const &rhs, int *result): lhs(lhs), rhs(rhs), result(result) { }
/// Equality check
CUTLASS_HOST_DEVICE
void operator()(TensorCoord const &coord) {
UnsignedType _lhs = reinterpret_cast<UnsignedType const &>(lhs.at(coord));
UnsignedType _rhs = reinterpret_cast<UnsignedType const &>(rhs.at(coord));
if (_lhs != _rhs) {
*result = 0;
}
}
};
} // namespace detail
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Returns true if two tensor views are equal.
template <typename ViewL, typename ViewR>
bool TensorEquals(ViewL const &lhs, ViewR const &rhs) {
// Sizes must be identical
if (lhs.size() != rhs.size()) {
return false;
}
// Allocate device memory to contain result of kernel reduction
HostTensor<int, 1> result(1);
result.fill(1);
result.sync_device();
typedef detail::TensorEqualsFunc<ViewL, ViewR> Func;
Func func(lhs, rhs, result.device_data());
TensorForEach<Func, ViewL::kRank, Func>(lhs.size(), func);
result.sync_host();
return result.at(0) != 0;
}
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Helper to apply a binary operator in place
template <typename ViewL, typename ViewR, typename BinaryFunc>
struct TensorFuncBinaryOp {
/// Coordinate in tensor's index space
typedef typename ViewL::TensorCoord TensorCoord;
//
// Data members
//
/// View of left-hand-side tensor
ViewL lhs;
/// View of right-hand-side tensor
ViewR rhs;
/// Binary function applied to each element
BinaryFunc func;
//
// Methods
//
/// Constructor
CUTLASS_HOST_DEVICE
TensorFuncBinaryOp(
ViewL const &lhs,
ViewR const &rhs,
BinaryFunc func = BinaryFunc()): lhs(lhs), rhs(rhs), func(func) { }
/// Equality check
CUTLASS_HOST_DEVICE
void operator()(TensorCoord const &coord) {
lhs.at(coord) = func(lhs.at(coord), rhs.at(coord));
}
};
///////////////////////////////////////////////////////////////////////////////////////////////////
namespace detail {
/// Helper to apply a binary operator in place
template <typename ViewL, typename ViewR>
struct TensorFillFunc {
/// Coordinate in tensor's index space
typedef typename ViewL::TensorCoord TensorCoord;
/// Destination element type
typedef typename ViewL::Storage DestType;
/// Source element type
typedef typename ViewR::Storage SrcType;
/// Parameters object
struct Params {
//
// Data members
//
/// View of left-hand-side tensor
ViewL lhs;
/// View of right-hand-side tensor
ViewR rhs;
/// Source offset coordinate
TensorCoord source_offset;
/// Size of the subtensor copied from the source
TensorCoord source_size;
/// Offset in destination
TensorCoord dest_offset;
//
// Methods
//
/// Constructs a parameters object for filling a tensor
Params(
ViewL const &lhs,
ViewR const &rhs,
TensorCoord const &source_offset = TensorCoord()
):
lhs(lhs), rhs(rhs), source_offset(source_offset), source_size(rhs.size() - source_offset) { }
/// Constructs a parameters object for filling a tensor
Params(
ViewL const &lhs,
ViewR const &rhs,
TensorCoord const &source_offset,
TensorCoord const &source_size,
TensorCoord const &dest_offset = TensorCoord()
):
lhs(lhs), rhs(rhs), source_offset(source_offset), source_size(source_size), dest_offset(dest_offset) { }
};
//
// Data members
//
Params params;
//
// Methods
//
/// Constructor
CUTLASS_HOST_DEVICE
TensorFillFunc(
Params const &params): params(params) { }
/// Equality check
CUTLASS_HOST_DEVICE
void operator()(TensorCoord const &coord) {
TensorCoord dst_coord = params.dest_offset + coord;
TensorCoord src_coord = params.source_offset + coord;
if (dst_coord < params.lhs.size() && src_coord < params.rhs.size()) {
params.lhs.at(dst_coord) = DestType(params.rhs.at(src_coord));
}
}
};
} // namespace detail
/// Fills a TensorView with the elements from another TensorView
template <typename ViewL, typename ViewR>
void TensorFill(
ViewL lhs,
ViewR rhs,
typename ViewL::TensorCoord const &source_offset,
typename ViewL::TensorCoord const &source_size,
typename ViewL::TensorCoord const &dest_offset) {
typedef typename ViewL::TensorCoord TensorCoord;
TensorCoord dst_size = lhs.size() - dest_offset;
TensorCoord src_size = rhs.size() - source_offset;
TensorCoord fill_size = dst_size.clamp(src_size);
// Fill function
typedef detail::TensorFillFunc<ViewL, ViewR> Func;
typedef typename Func::Params Params;
Params params(lhs, rhs, source_offset, source_size, dest_offset);
TensorForEach<Func, ViewL::kRank, Params>(fill_size, params);
}
/// Fills a TensorView with the elements from another TensorView
template <typename ViewL, typename ViewR>
void TensorFill(
ViewL lhs,
ViewR rhs,
typename ViewL::TensorCoord const &source_offset = typename ViewL::TensorCoord()) {
typedef typename ViewL::TensorCoord TensorCoord;
TensorFill(lhs, rhs, source_offset, rhs.size(), TensorCoord());
}
///////////////////////////////////////////////////////////////////////////////////////////////////
namespace detail {
/// Helper to apply a binary operator in place
template <typename ViewL>
struct TensorFillElementFunc {
/// Coordinate in tensor's index space
typedef typename ViewL::TensorCoord TensorCoord;
/// Destination element type
typedef typename ViewL::Storage DestType;
/// Parameters object
struct Params {
//
// Data members
//
/// View of left-hand-side tensor
ViewL lhs;
/// Source offset coordinate
TensorCoord offset;
/// Element to overwrite with
DestType value;
//
// Methods
//
/// Constructs a parameters object for filling a tensor
CUTLASS_HOST_DEVICE
Params(
ViewL const &lhs,
DestType const &value,
TensorCoord const &offset = TensorCoord()
):
lhs(lhs), value(value), offset(offset) { }
};
//
// Data members
//
Params params;
//
// Methods
//
/// Constructor
CUTLASS_HOST_DEVICE
TensorFillElementFunc(
Params const &params): params(params) { }
/// Equality check
CUTLASS_HOST_DEVICE
void operator()(TensorCoord const &coord) {
TensorCoord dst_coord = params.offset + coord;
if (dst_coord < params.size) {
params.lhs.at(dst_coord) = params.value;
}
}
};
} // namespace detail
/// Method to perform the actual fill
template <typename ViewL>
void TensorFillElement(
ViewL const &lhs,
typename ViewL::Storage const &value,
typename ViewL::TensorCoord const &offset,
typename ViewL::TensorCoord const &size) {
// Fill function
typedef detail::TensorFillElementFunc<ViewL> Func;
typedef typename Func::Params Params;
Params params(lhs, value, offset);
TensorForEach<Func, ViewL::kRank, Params>(size, params);
}
/// Fills a tensor
template <typename ViewL>
void TensorFillElement(
ViewL lhs,
typename ViewL::Storage value,
typename ViewL::TensorCoord const &offset =typename ViewL::Storage()) {
TensorFillElement(lhs, value, offset, lhs.size() - offset);
}
/// Constructs a parameters object for filling a tensor
template <typename ViewL>
void TensorFillElement(
ViewL lhs,
typename ViewL::Storage value,
typename ViewL::Storage const &offset,
typename ViewL::Storage const &size) {
TensorFillElement(lhs, value, offset, size);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace device
} // namespace reference
} // namespace cutlass
tools/util/reference/device/tensor_foreach.h
+72
View File
@@ -0,0 +1,72 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#include <stdexcept>
#include "cutlass/cutlass.h"
#include "tools/util/reference/device/kernel/tensor_foreach.h"
namespace cutlass {
namespace reference {
namespace device {
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Launches a kernel for each element in a tensor's index space.
template <typename Func, int Rank, typename Params>
struct TensorForEach {
/// Constructor performs the operation.
TensorForEach(Coord<Rank> size, Params params = Params(), int grid_size = 0, int block_size = 0) {
if (!grid_size || !block_size) {
// if grid_size or block_size are zero, query occupancy using the CUDA Occupancy API
cudaError_t result = cudaOccupancyMaxPotentialBlockSize(
&grid_size,
&block_size,
reinterpret_cast<void const *>(kernel::TensorForEach<Func, Rank, Params>));
if (result != cudaSuccess) {
throw std::runtime_error("Failed to query occupancy.");
}
// Limit block size. This has the effect of increasing the number of items processed by a
// single thread and reduces the impact of initialization overhead.
block_size = (block_size < 128 ? block_size : 128);
}
dim3 grid(grid_size, 1, 1);
dim3 block(block_size, 1, 1);
kernel::TensorForEach<Func, Rank, Params><<< grid, block >>>(size, params);
}
};
///////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace device
} // namespace reference
} // namesace cutlass
tools/util/reference/host/gemm.h
+270
View File
@@ -0,0 +1,270 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Reference implementation for GEMM in host-side code.
*/
#pragma once
#include "cutlass/coord.h"
#include "cutlass/matrix_traits.h"
#include "cutlass/tensor_view.h"
#include "cutlass/gemm/gemm_coord.h"
namespace cutlass {
namespace reference {
namespace host {
////////////////////////////////////////////////////////////////////////////////////////////////////
namespace detail {
/// Template function to compute an inner product.
template <typename Atype, typename Btype, typename Ctype>
Ctype inner_product(Atype a, Btype b, Ctype c) {
return Ctype(a) * Ctype(b) + c;
}
/// Specialization for matrix multiplication with binary operands
template <>
inline int inner_product<Vector<bin1_t, 32>, Vector<bin1_t, 32>, int>(
Vector<bin1_t, 32> a,
Vector<bin1_t, 32> b,
int c) {
int accum = 0;
for (int bit = 0; bit < 32; bit++) {
accum += a[bit] ^ b[bit];
}
return accum + c;
}
/// Specialization for matrix multiplication with signed 4-bit integer operands
template <> inline
int inner_product<Vector<int4_t, 8>, Vector<int4_t, 8>, int>(
Vector<int4_t, 8> a,
Vector<int4_t, 8> b,
int c) {
int accum = 0;
for (int k = 0; k < 8; k++) {
accum += a[k] * b[k];
}
return accum + c;
}
/// Specialization for matrix multiplication with unsigned 4-bit integer operands
template <> inline
int inner_product<Vector<uint4_t, 8>, Vector<uint4_t, 8>, int>(
Vector<uint4_t, 8> a,
Vector<uint4_t, 8> b,
int c) {
int accum = 0;
for (int k = 0; k < 8; k++) {
accum += a[k] * b[k];
}
return accum + c;
}
////////////////////////////////////////////////////////////////////////////////////////////////////
template <typename SrcType, typename DstType>
struct Cast {
// Default behavior: convert to the destination type
static inline DstType apply(SrcType src) { return static_cast<DstType>(src); };
};
template <>
struct Cast<float, int8_t> {
static inline int8_t apply(float src) {
// Clamp to the range of signed 8-bit integers.
return static_cast<int8_t>(fmaxf(-128.f, fminf(127.f, src)));
};
};
template <>
struct Cast<float, uint8_t> {
static inline uint8_t apply(float src) {
// Clamp to the range of signed 8-bit integers.
return static_cast<uint8_t>(fmaxf(0.f, fminf(255.f, src)));
};
};
} // namespace detail
////////////////////////////////////////////////////////////////////////////////////////////////////
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
/// objects.
///
/// Explicitly naming types needed by this template can be cumbersome, particularly for the
/// accumulator type, so a function argument 'initial_accum' is exposed. Passing
/// AccumulatorType(0) as the last function argument can be easier than naming all template
/// arguments explicitly.
template <
typename TensorRefA,
typename TensorRefB,
typename TensorRefC,
typename ScalarType,
typename AccumulatorType
>
void Gemm(
gemm::GemmCoord problem_size,
ScalarType alpha,
TensorRefA tensor_a,
TensorRefB tensor_b,
ScalarType beta,
TensorRefC tensor_c,
AccumulatorType initial_accum) {
typedef typename TensorRefA::Storage AType;
typedef typename TensorRefB::Storage BType;
typedef typename TensorRefC::Storage CType;
static_assert(
TensorRefA::kRank == 2 &&
TensorRefB::kRank == 2 &&
TensorRefC::kRank == 2, "Tensors must be of rank 2");
// Note: batch is ignored.
int const M = problem_size.m();
int const N = problem_size.n();
int const K = problem_size.k();
// Blocking necessary to speedup reference implementation
int const Mblock = 32;
int const Nblock = 32;
for (int row_block = 0; row_block < M; row_block += Mblock) {
for (int col_block = 0; col_block < N; col_block += Nblock) {
AccumulatorType accum[Mblock][Nblock];
for (int j = 0; j < Nblock; j++) {
for (int i = 0; i < Mblock; i++) {
accum[i][j] = initial_accum;
}
}
for (int k_block = 0; k_block < K; ++k_block) {
for (int j = 0; j < Nblock; j++) {
for (int i = 0; i < Mblock; i++) {
int row = row_block + i;
int col = col_block + j;
if (row < M && col < N) {
AType a = tensor_a.at(MatrixCoord(row, k_block));
BType b = tensor_b.at(MatrixCoord(k_block, col));
accum[i][j] = detail::inner_product(a, b, accum[i][j]);
}
}
}
}
for (int j = 0; j < Nblock; j++) {
for (int i = 0; i < Mblock; i++) {
int row = row_block + i;
int col = col_block + j;
MatrixCoord coord = MatrixCoord(row, col);
if (row < M && col < N) {
tensor_c.at(coord) = detail::Cast<ScalarType, CType>::apply(
alpha * ScalarType(accum[i][j]) +
beta * ScalarType(tensor_c.at(coord)));
}
}
}
}
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
/// objects.
///
/// This assumes the accumulator type is the same type as the scalars.
template <
typename TensorRefA,
typename TensorRefB,
typename TensorRefC,
typename ScalarType
>
void Gemm(
gemm::GemmCoord problem_size,
ScalarType alpha,
TensorRefA tensor_a,
TensorRefB tensor_b,
ScalarType beta,
TensorRefC tensor_c) {
Gemm(problem_size, alpha, tensor_a, tensor_b, beta, tensor_c, ScalarType(0));
}
////////////////////////////////////////////////////////////////////////////////////////////////////
/// Computes a batch of GEMMs over a set of matrices of common dimension.
template <
typename TensorRefCollectionA,
typename TensorRefCollectionB,
typename TensorRefCollectionC,
typename ScalarType,
typename AccumulatorType
>
void BatchGemm(
gemm::GemmCoord problem_size,
ScalarType alpha,
TensorRefCollectionA const& tensor_a,
TensorRefCollectionB const& tensor_b,
ScalarType beta,
TensorRefCollectionC &tensor_c,
AccumulatorType initial_accum = AccumulatorType(0)) {
typename TensorRefCollectionA::ConstIterator tensor_a_it = tensor_a.begin();
typename TensorRefCollectionB::ConstIterator tensor_b_it = tensor_b.begin();
typename TensorRefCollectionC::ConstIterator tensor_c_it = tensor_c.begin();
for (int batch = 0;
batch < problem_size.batch();
++batch, ++tensor_a_it, ++tensor_b_it, ++tensor_c_it) {
Gemm(
problem_size,
alpha,
*tensor_a_it,
*tensor_b_it,
beta,
*tensor_c_it,
initial_accum);
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace host
} // namespace reference
} // namespace cutlass
tools/util/reference/host/tensor_elementwise.h
+478
View File
@@ -0,0 +1,478 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Defines host-side elementwise operations on TensorView.
*/
#pragma once
// Standard Library includes
#include <fstream>
#include <ostream>
#include <stdexcept>
#include <string>
#include <utility>
#include <cstdlib>
#include <cmath>
// Cutlass includes
#include "cutlass/cutlass.h"
#include "tools/util/distribution.h"
#include "tools/util/type_traits.h"
#include "tools/util/reference/host/tensor_foreach.h"
namespace cutlass {
namespace reference {
namespace host {
///////////////////////////////////////////////////////////////////////////////////////////////////
namespace detail {
/// Computes a random uniform distribution
template <typename View_>
struct RandomUniformFunc {
/// View type
typedef View_ View;
/// Scalar type
typedef typename View::Storage T;
/// Coordinate in tensor's index space
typedef typename View::TensorCoord TensorCoord;
/// Parameters structure
struct Params {
/// View object
View view;
/// RNG seed
unsigned seed;
/// Distriubtion
Distribution dist;
/// Default ctor
Params() { }
/// Constructor
Params(
View const &view,
unsigned seed,
Distribution dist
): view(view), seed(seed), dist(dist) { }
};
//
// Data members
//
/// Parameters object
Params params;
//
// Methods
//
/// Device-side initialization of RNG
RandomUniformFunc(Params const &params): params(params) {
std::srand(params.seed);
}
/// Compute random value and update RNG state
void operator()(TensorCoord const &coord) {
double range = params.dist.uniform.max - params.dist.uniform.min;
double rnd = double(std::rand()) / double(RAND_MAX);
rnd = params.dist.uniform.min + range * rnd;
// Random values are cast to integer after scaling by a power of two to facilitate error
// testing
T result;
if (params.dist.int_scale >= 0) {
rnd = double(int(rnd * double(1 << params.dist.int_scale)));
result = T(rnd / double(1 << params.dist.int_scale));
}
else {
result = T(rnd);
}
params.view.at(coord) = result;
}
};
/// Computes a random Gaussian distribution
template <typename View_>
struct RandomGaussianFunc {
/// View type
typedef View_ View;
/// Scalar type
typedef typename View::Storage T;
/// Coordinate in tensor's index space
typedef typename View::TensorCoord TensorCoord;
/// Parameters structure
struct Params {
/// View object
View view;
/// RNG seed
unsigned seed;
/// RNG distribution
Distribution dist;
/// Default ctor
Params() { }
/// Constructor
Params(
View const &view,
unsigned seed,
Distribution dist
): view(view), seed(seed), dist(dist) { }
};
//
// Data members
//
/// Parameters object
Params params;
/// Constant PI
double pi;
//
// Methods
//
/// Device-side initialization of RNG
RandomGaussianFunc(Params const &params): params(params) {
pi = std::acos(-1);
}
/// Compute random value and update RNG state
void operator()(TensorCoord const &coord) {
// Box-Muller transform to generate random numbers with Normal distribution
double u1 = double(std::rand()) / double(RAND_MAX);
double u2 = double(std::rand()) / double(RAND_MAX);
double rnd = std::sqrt(-2 * std::log(u1)) * std::cos(2 * pi * u2);
// Scale according to Gaussian distribution parameters
rnd = params.dist.gaussian.mean + params.dist.gaussian.stddev * rnd;
T result;
if (params.dist.int_scale >= 0) {
rnd = double(int(rnd * double(1 << params.dist.int_scale)));
result = T(rnd / double(1 << params.dist.int_scale));
}
else {
result = T(rnd);
}
params.view.at(coord) = result;
}
};
/// Computes a linear combination of each element
template <typename View_>
struct LinearCombinationFunc {
/// View type
typedef View_ View;
/// Scalar type
typedef typename View::Storage T;
/// Coordinate in tensor's index space
typedef typename View::TensorCoord TensorCoord;
//
// Data members
//
/// TensorView object
View view;
/// Delta
Coord<View::kRank, double> delta;
/// Offset
double offset;
//
// Methods
//
/// Constructor
LinearCombinationFunc(
View const &view,
Distribution dist
): view(view) {
offset = dist.linear.offset;
if (View::kRank >= 1) {
delta[View::kRank - 1] = dist.linear.delta_column;
}
if (View::kRank >= 2) {
delta[View::kRank - 2] = dist.linear.delta_row;
}
// Additional ranks have delta of zero
for (int i = View::kRank - 2; i > 0; --i) {
delta[i - 1] = 0;
}
}
/// Compute linear combination
void operator()(TensorCoord const &coord) {
double result = offset;
for (int i = 0; i < View::kRank; ++i) {
result += delta[i] * double(coord[i]);
}
view.at(coord) = T(result);
}
};
/// Returns 1 or 0 if the coordinate is along the tensor's diagonal
template <typename View_>
struct IdentityFunc {
/// TensorView
typedef View_ View;
/// Scalar type
typedef typename View::Storage T;
/// Coordinate in tensor's index space
typedef typename View::TensorCoord TensorCoord;
//
// Data members
//
/// View object
View view;
/// Default ctor
IdentityFunc(View const &view): view(view) { }
/// Computes an identity
void operator()(TensorCoord const &coord) {
bool equal = true;
for (int i = 0; i < View::kRank; ++i) {
if (coord[i] != coord[0]) {
equal = false;
}
}
view.at(coord) = equal ? T(1) : T(0);
}
};
} // namespace detail
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Initializes a tensor randomly or procedurally.
template <typename View>
void TensorInitialize(View const &view,
unsigned seed,
Distribution const &dist) {
typedef typename View::Storage Scalar;
switch (dist.kind) {
case Distribution::Uniform:
{
typedef detail::RandomUniformFunc<View> Func;
typedef typename Func::Params Params;
TensorForEach<Func, View::kRank, Params>(
view.size(),
Params(view, seed, dist)
);
}
break;
case Distribution::Gaussian:
{
typedef detail::RandomGaussianFunc<View> Func;
typedef typename Func::Params Params;
TensorForEach<Func, View::kRank, Params>(
view.size(),
Params(view, seed, dist)
);
}
break;
case Distribution::Linear:
{
typedef detail::LinearCombinationFunc<View> Func;
TensorForEach<Func, View::kRank, Func>(
view.size(),
Func(view, dist));
}
break;
case Distribution::Identity:
{
typedef detail::IdentityFunc<View> Func;
Func func(view);
TensorForEach<Func, View::kRank, Func>(view.size(), func);
}
break;
default:
break;
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////
namespace detail {
/// Compares two tensor views of equal rank and dimension.
template <typename ViewL, typename ViewR>
struct TensorEqualsFunc {
/// Storage type
typedef typename ViewL::Storage T;
/// Unsigned integer type of same size as View type
typedef typename cutlass::TypeTraits<T>::unsigned_type UnsignedType;
/// Coordinate in tensor's index space
typedef typename ViewL::TensorCoord TensorCoord;
/// Assertions
static_assert(ViewL::kRank == ViewR::kRank,
"Cannot compare tensors of different rank");
//
// Data members
//
/// View of left-hand-side tensor
ViewL lhs;
/// View of right-hand-side tensor
ViewR rhs;
/// Pointer to result scalar - only written with 0 if values are incorrect
int *result;
//
// Methods
//
/// Constructor
TensorEqualsFunc(ViewL const &lhs, ViewR const &rhs, int *result): lhs(lhs), rhs(rhs), result(result) { }
/// Equality check
void operator()(TensorCoord const &coord) {
UnsignedType _lhs = reinterpret_cast<UnsignedType const &>(lhs.at(coord));
UnsignedType _rhs = reinterpret_cast<UnsignedType const &>(rhs.at(coord));
if (_lhs != _rhs) {
*result = 0;
}
}
};
} // namespace detail
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Returns true if two tensor views are equal.
template <typename ViewL, typename ViewR>
bool TensorEquals(ViewL const &lhs, ViewR const &rhs) {
// Sizes must be identical
if (lhs.size() != rhs.size()) {
return false;
}
int result = 1;
typedef detail::TensorEqualsFunc<ViewL, ViewR> Func;
Func func(lhs, rhs, &result);
TensorForEach<Func, ViewL::kRank, Func>(lhs.size(), func);
return result != 0;
}
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Helper to apply a binary operator in place
template <typename ViewL, typename ViewR, typename BinaryFunc>
struct TensorFuncBinaryOp {
/// Coordinate in tensor's index space
typedef typename ViewL::TensorCoord TensorCoord;
//
// Data members
//
/// View of left-hand-side tensor
ViewL lhs;
/// View of right-hand-side tensor
ViewR rhs;
/// Binary function applied to each element
BinaryFunc func;
//
// Methods
//
/// Constructor
TensorFuncBinaryOp(
ViewL const &lhs,
ViewR const &rhs,
BinaryFunc func = BinaryFunc()): lhs(lhs), rhs(rhs), func(func) { }
/// Equality check
void operator()(TensorCoord const &coord) {
lhs.at(coord) = func(lhs.at(coord), rhs.at(coord));
}
};
///////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace host
} // namespace reference
} // namespace cutlass
tools/util/reference/host/tensor_foreach.h
+102
View File
@@ -0,0 +1,102 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#include <stdexcept>
#include "cutlass/cutlass.h"
#include "tools/util/reference/device/kernel/tensor_foreach.h"
namespace cutlass {
namespace reference {
namespace host {
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Defines several helpers
namespace detail {
/// Helper to perform for-each operation
template <typename Func, int Rank, int RankRemaining>
struct TensorForEachHelper {
/// Index of the active rank
static int const kActiveRank = Rank - RankRemaining - 1;
/// Constructor for general rank
TensorForEachHelper(
Func &func,
Coord<Rank> const &size,
Coord<Rank> &coord) {
for (int i = 0; i < size.at(kActiveRank); ++i) {
coord[kActiveRank] = i;
TensorForEachHelper<Func, Rank, RankRemaining - 1>(func, size, coord);
}
}
};
/// Helper to perform for-each operation
template <typename Func, int Rank>
struct TensorForEachHelper<Func, Rank, 0> {
/// Index of the active rank
static int const kActiveRank = Rank - 1;
/// Constructor for fastest chaning rank
TensorForEachHelper(
Func &func,
Coord<Rank> const &size,
Coord<Rank> &coord) {
for (int i = 0; i < size.at(kActiveRank); ++i) {
coord[kActiveRank] = i;
func(coord);
}
}
};
} // namespace detail
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Iterates over the index space of a tensor
template <typename Func, int Rank, typename Params>
struct TensorForEach {
/// Constructor performs the operation.
TensorForEach(Coord<Rank> size, Params params = Params()) {
Func func(params);
Coord<Rank> coord;
detail::TensorForEachHelper<Func, Rank, Rank - 1>(func, size, coord);
}
};
///////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace host
} // namespace reference
} // namespace cutlass
tools/util/tensor_view_io.h
+124 -27
View File
@@ -24,38 +24,135 @@
**************************************************************************************************/
#pragma once
#include <cutlass/core_io.h>
#include <cutlass/tensor_view.h>
#include "cutlass/core_io.h"
#include "cutlass/tensor_view.h"
template <typename T>
inline std::ostream& tensor_view_output(std::ostream& out, T t) {
out << t;
return out;
}
namespace cutlass {
template <>
inline std::ostream& tensor_view_output<int8_t>(std::ostream& out, int8_t t) {
out << int(t);
return out;
}
///////////////////////////////////////////////////////////////////////////////////////////////////
template <typename T>
inline std::ostream& operator<<(std::ostream& out, cutlass::TensorView<T> const& tensor) {
for (int batch = 0; batch < tensor.size(0); ++batch) {
out << "[\n ";
for (int h = 0; h < tensor.size(1); ++h) {
for (int w = 0; w < tensor.size(2); ++w) {
for (int c = 0; c < tensor.size(3); ++c) {
out << ((c | w) ? ", " : "");
tensor_view_output(out, tensor.at(cutlass::make_Coord(batch, h, w, c)));
}
}
if (h + 1 < tensor.size(1)) {
out << " ;\n ";
}
namespace detail {
/// Helper to write the least significant rank of a TensorView
template <
typename Storage_,
int Rank_,
typename MapFunc_,
int StorageRank_,
typename Index_,
typename LongIndex_
>
inline std::ostream & TensorView_WriteLeastSignificantRank(
std::ostream& out,
cutlass::TensorView<
Storage_,
Rank_,
MapFunc_,
StorageRank_,
Index_,
LongIndex_> const& tensor,
cutlass::Coord<Rank_> const &start_coord,
int rank,
std::streamsize width) {
for (int idx = 0; idx < tensor.size(rank); ++idx) {
Coord<Rank_> coord(start_coord);
coord[rank] = idx;
if (idx) {
out.width(0);
out << ", ";
}
out << " ]";
if (idx || coord) {
out.width(width);
}
out << ScalarIO<Storage_>(tensor.at(coord));
}
return out;
}
/// Helper to write a rank of a TensorView
template <
typename Storage_,
int Rank_,
typename MapFunc_,
int StorageRank_,
typename Index_,
typename LongIndex_
>
inline std::ostream & TensorView_WriteRank(
std::ostream& out,
cutlass::TensorView<
Storage_,
Rank_,
MapFunc_,
StorageRank_,
Index_,
LongIndex_> const& tensor,
cutlass::Coord<Rank_> const &start_coord,
int rank,
std::streamsize width) {
// If called on the least significant rank, write the result as a row
if (rank + 1 == Rank_) {
return TensorView_WriteLeastSignificantRank(out, tensor, start_coord, rank, width);
}
// Otherwise, write a sequence of rows and newlines
for (int idx = 0; idx < tensor.size(rank); ++idx) {
Coord<Rank_> coord(start_coord);
coord[rank] = idx;
if (rank + 2 == Rank_) {
// Write least significant ranks asa matrix with rows delimited by ";\n"
out << (idx ? ";\n" : "");
TensorView_WriteLeastSignificantRank(out, tensor, coord, rank + 1, width);
}
else {
// Higher ranks are separated by newlines
out << (idx ? "\n" : "");
TensorView_WriteRank(out, tensor, coord, rank + 1, width);
}
}
return out;
}
} // namespace detail
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Prints human-readable representation of a TensorView to an ostream
template <
typename Storage_,
int Rank_,
typename MapFunc_,
int StorageRank_,
typename Index_,
typename LongIndex_
>
inline std::ostream& operator<<(
std::ostream& out,
TensorView<
Storage_,
Rank_,
MapFunc_,
StorageRank_,
Index_,
LongIndex_> const& tensor) {
// Prints a TensorView according to the following conventions:
// - least significant rank is printed as rows separated by ";\n"
// - all greater ranks are delimited with newlines
//
// The result is effectively a whitespace-delimited series of 2D matrices.
return detail::TensorView_WriteRank(out, tensor, Coord<Rank_>(), 0, out.width());
}
///////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace cutlass
tools/util/type_traits.h
+110 -1
View File
@@ -33,12 +33,52 @@
#include <stdint.h>
#include "half.h"
#include "cutlass/vector.h"
#include "cutlass/util/complex.h"
namespace cutlass {
struct half_t;
template <typename T>
struct TypeTraits;
struct TypeTraits {
typedef T host_type;
typedef T device_type;
static inline T remove_negative_zero(T x) { return x; }
static inline T to_print(T x) { return x; }
};
template <>
struct TypeTraits<Vector<bin1_t, 32> > {
static cudaDataType_t const cublas_type = CUDA_R_32I;
typedef Vector<bin1_t, 32> host_type;
typedef Vector<bin1_t, 32> device_type;
typedef uint32_t integer_type;
typedef uint32_t unsigned_type;
static inline uint32_t remove_negative_zero(uint32_t x) { return x; }
static inline uint32_t to_print(uint32_t x) { return x; }
};
template <>
struct TypeTraits< Vector<int4_t, 8> > {
static cudaDataType_t const cublas_type = CUDA_R_32I;
typedef Vector<int4_t, 8> host_type;
typedef Vector<int4_t, 8> device_type;
typedef uint32_t integer_type;
typedef uint32_t unsigned_type;
static inline uint32_t remove_negative_zero(uint32_t x) { return x; }
static inline uint32_t to_print(uint32_t x) { return x; }
};
template <>
struct TypeTraits< Vector<uint4_t, 8> > {
static cudaDataType_t const cublas_type = CUDA_R_32I;
typedef Vector<uint4_t, 8> host_type;
typedef Vector<uint4_t, 8> device_type;
typedef uint32_t integer_type;
typedef uint32_t unsigned_type;
static inline uint32_t remove_negative_zero(uint32_t x) { return x; }
static inline uint32_t to_print(uint32_t x) { return x; }
};
template <>
struct TypeTraits<int8_t> {
@@ -158,4 +198,73 @@ struct TypeTraits<double> {
static inline double remove_negative_zero(double x) { return x == -0.0 ? 0.0 : x; }
static inline double to_print(double x) { return x; }
};
///////////////////////////////////////////////////////////////////////////////////////////////////
//
// Complex types
//
///////////////////////////////////////////////////////////////////////////////////////////////////
template <>
struct TypeTraits<platform::complex<half> > {
static cudaDataType_t const cublas_type = CUDA_C_16F;
typedef platform::complex<half_t> host_type;
typedef platform::complex<half> device_type;
typedef int16_t integer_type;
typedef uint16_t unsigned_type;
};
template <>
struct TypeTraits<platform::complex<half_t> > {
static cudaDataType_t const cublas_type = CUDA_C_16F;
typedef platform::complex<half_t> host_type;
typedef platform::complex<half> device_type;
typedef int16_t integer_type;
typedef uint16_t unsigned_type;
static inline platform::complex<half_t> remove_negative_zero(platform::complex<half_t> x) {
return platform::complex<half_t>(
real(x) == -0.f ? half_t(0) : real(x),
imag(x) == -0.f ? half_t(0) : imag(x)
);
}
static inline platform::complex<half_t> to_print(platform::complex<half_t> x) { return x; }
};
template <>
struct TypeTraits<platform::complex<float> > {
static cudaDataType_t const cublas_type = CUDA_C_32F;
typedef platform::complex<float> host_type;
typedef platform::complex<float> device_type;
typedef int64_t integer_type;
typedef uint64_t unsigned_type;
static inline platform::complex<float> remove_negative_zero(platform::complex<float> x) {
return platform::complex<float>(
real(x) == -0.f ? 0.f : real(x),
imag(x) == -0.f ? 0.f : imag(x)
);
}
static inline platform::complex<float> to_print(platform::complex<float> x) { return x; }
};
template <>
struct TypeTraits<platform::complex<double> > {
static cudaDataType_t const cublas_type = CUDA_C_64F;
typedef platform::complex<double> host_type;
typedef platform::complex<double> device_type;
struct integer_type { int64_t real, imag; };
struct unsigned_type { uint64_t real, imag; };
static inline platform::complex<double> remove_negative_zero(platform::complex<double> x) {
return platform::complex<double>(
real(x) == -0.0 ? 0.0 : real(x),
imag(x) == -0.0 ? 0.0 : imag(x)
);
}
static inline platform::complex<double> to_print(platform::complex<double> x) { return x; }
};
///////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace cutlass