leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 29 Packages Projects Releases Wiki Activity

CUTLASS 3.6.0 (#1850)

Browse Source 
* v3.6

* update changelog

* update readme

* fix typo

* fixing typos

* hopper gemm with weight prefetch

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
Yujia Zhai
2024-10-09 15:33:27 -04:00
committed by GitHub
co-authored by yuzhai Haicheng Wu
parent 0837a2a00a
commit cc3c29a81a
354 changed files with 105937 additions and 8197 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/util/include/cutlass/util/device_memory.h
+41 -2
View File
@@ -37,9 +37,11 @@
*/
#include <memory>
#include <sstream>
#include "cutlass/platform/platform.h"
#include "cutlass/numeric_types.h"
#include "cutlass/trace.h"
#include "exceptions.h"
namespace cutlass {
@@ -61,8 +63,20 @@ T* allocate(size_t count = 1) {
cudaError_t cuda_error = cudaMalloc((void**)&ptr, bytes);
if (cuda_error != cudaSuccess) {
#if (CUTLASS_DEBUG_TRACE_LEVEL > 0)
std::ostringstream os;
os << "cutlass::device_memory::allocate: cudaMalloc failed: bytes=" << bytes;
CUTLASS_TRACE_HOST(os.str());
#endif
throw cuda_exception("Failed to allocate memory", cuda_error);
}
#if (CUTLASS_DEBUG_TRACE_LEVEL > 1)
else {
std::ostringstream os;
os << "cutlass::device_memory::allocate: Successful cudaMalloc: bytes=" << bytes;
CUTLASS_TRACE_HOST(os.str());
}
#endif
return ptr;
}
@@ -85,11 +99,36 @@ void free(T* ptr) {
template <typename T>
void copy(T* dst, T const* src, size_t count, cudaMemcpyKind kind) {
size_t bytes = count * sizeof_bits<T>::value / 8;
if (bytes == 0 && count > 0)
if (bytes == 0 && count > 0) {
bytes = 1;
}
cudaError_t cuda_error = (cudaMemcpy(dst, src, bytes, kind));
if (cuda_error != cudaSuccess) {
throw cuda_exception("cudaMemcpy() failed", cuda_error);
std::ostringstream os;
os << "cutlass::device_memory::copy: cudaMemcpy() failed: "
<< "dst=" << dst << ", src=" << src
<< ", bytes=" << bytes << ", count=" << count;
if (kind == cudaMemcpyHostToDevice) {
os << ", kind=cudaMemcpyHostToDevice";
}
else if (kind == cudaMemcpyDeviceToHost) {
os << ", kind=cudaMemcpyDeviceToHost";
}
else if (kind == cudaMemcpyDeviceToDevice) {
os << ", kind=cudaMemcpyDeviceToDevice";
}
else if (kind == cudaMemcpyHostToHost) {
os << ", kind=cudaMemcpyHostToHost";
}
else if (kind == cudaMemcpyDefault) {
os << ", kind=cudaMemcpyDefault";
}
else {
os << ", kind=Unknown";
}
os << ", error: " << cudaGetErrorString(cuda_error);
throw cuda_exception(os.str().c_str(), cuda_error);
}
}
tools/util/include/cutlass/util/distribution.h
+8 -4
View File
@@ -51,6 +51,8 @@ struct Distribution {
struct {
double min;
double max;
// Percent elements set to NaN
double pnan;
} uniform;
/// Gaussian distribution
@@ -82,17 +84,18 @@ struct Distribution {
Distribution() : kind(Invalid), int_scale(0) {}
/// Configures distribution as uniform random
Distribution &set_uniform(double _min, double _max, int _int_scale = 0) {
/// Configures distribution as uniform random
Distribution &set_uniform(double _min, double _max, int _int_scale = 0, double _pnan = 0) {
kind = Uniform;
uniform.min = _min;
uniform.max = _max;
int_scale = _int_scale;
uniform.pnan = _pnan;
return *this;
}
/// Configures distribution as Gaussian distribution
Distribution &set_gaussian(double _mean, double _stddev, int _int_scale = 0, double _pnz = 100.0) {
Distribution &set_gaussian(double _mean, double _stddev, int _int_scale = 0, double _pnz = 1.0) {
kind = Gaussian;
gaussian.mean = _mean;
gaussian.stddev = _stddev;
@@ -125,7 +128,8 @@ struct Distribution {
inline std::ostream &operator<<(std::ostream &out, cutlass::Distribution const &dist) {
switch (dist.kind) {
case cutlass::Distribution::Uniform:
out << "uniform, min: " << dist.uniform.min << ", max: " << dist.uniform.max;
out << "uniform, min: " << dist.uniform.min << ", max: " << dist.uniform.max
<< ", pnan: " << dist.uniform.pnan;
break;
case cutlass::Distribution::Gaussian:
out << "gaussian, mean: " << dist.gaussian.mean << ", stddev: " << dist.gaussian.stddev
tools/util/include/cutlass/util/host_tensor.h
+10 -1
View File
@@ -177,16 +177,25 @@ public:
void reserve(
size_t count, ///< size of tensor in elements
bool device_backed_ = true) { ///< if true, device memory is also allocated
#if (CUTLASS_DEBUG_TRACE_LEVEL > 1)
CUTLASS_TRACE_HOST("cutlass::HostTensor::reserve(count=" << count << ", device_backed_=" << (device_backed_ ? "true" : "false") << ")");
#endif
device_.reset();
host_.clear();
size_t count_container = count_to_container_storage_unit_count(count);
#if (CUTLASS_DEBUG_TRACE_LEVEL > 1)
CUTLASS_TRACE_HOST("cutlass::HostTensor::reserve: host_.resize(" << count_container << ")");
#endif
host_.resize(count_container);
// Allocate memory
StorageUnit* device_memory = nullptr;
if (device_backed_) {
#if (CUTLASS_DEBUG_TRACE_LEVEL > 1)
CUTLASS_TRACE_HOST("cutlass::HostTensor::reserve: device_memory::allocate(" << count_container << ")");
#endif
device_memory = device_memory::allocate<StorageUnit>(count_container);
}
device_.reset(device_memory, device_backed_ ? count_container : 0);
@@ -394,7 +403,7 @@ public:
void sync_device() {
if (device_backed()) {
device_memory::copy_to_device(
device_.get(), host_.data(), host_.capacity());
device_.get(), host_.data(), host_.size());
}
}
tools/util/include/cutlass/util/packed_stride.hpp
+2
View File
@@ -35,6 +35,8 @@
#pragma once
#include "cute/layout.hpp"
#include "cute/container/array.hpp" // cute::array
#include "cutlass/conv/convolution.h" // cutlass::conv::Operator
/////////////////////////////////////////////////////////////////////////////////////////////////
tools/util/include/cutlass/util/reference/device/tensor_fill.h
+113 -14
View File
@@ -57,6 +57,7 @@
#include "cutlass/complex.h"
#include "cutlass/tensor_view.h"
#include "cutlass/blas3.h"
#include "cutlass/numeric_types.h"
#include "cutlass/layout/vector.h"
@@ -117,6 +118,7 @@ struct RandomGaussianFunc {
int int_scale;
FloatType float_scale_up;
FloatType float_scale_down;
int exclude_zero; ///< If non-negative, excludes zeros
//
// Methods
@@ -127,12 +129,14 @@ struct RandomGaussianFunc {
uint64_t seed_ = 0,
Element mean_ = 0,
Element stddev_ = 1,
int int_scale_ = -1
int int_scale_ = -1,
int exclude_zero_ = -1
):
seed(seed_),
mean(static_cast<FloatType>(mean_)),
stddev(static_cast<FloatType>(stddev_)),
int_scale(int_scale_) {
int_scale(int_scale_),
exclude_zero(exclude_zero_) {
float_scale_up = FloatType(IntType(2) << int_scale); // scale up to clamp low order bits
float_scale_down = FloatType(1) / FloatType(IntType(2) << int_scale);
@@ -178,6 +182,15 @@ struct RandomGaussianFunc {
result = Element(rnd);
}
if (params.exclude_zero >=0 && result == Element(0.0)) {
if (rnd > FloatType(0)) {
rnd += FloatType(1);
} else {
rnd -= FloatType(1);
}
result = Element(rnd);
}
return result;
}
};
@@ -203,6 +216,7 @@ struct RandomGaussianFunc<complex<Real>> {
int int_scale;
FloatType float_scale_up;
FloatType float_scale_down;
int exclude_zero; ///< If non-negative, excludes zeros
//
// Methods
@@ -213,12 +227,14 @@ struct RandomGaussianFunc<complex<Real>> {
uint64_t seed_ = 0,
Real mean_ = 0,
Real stddev_ = 1,
int int_scale_ = -1
int int_scale_ = -1,
int exclude_zero_ = -1
):
seed(seed_),
mean(static_cast<FloatType>(mean_)),
stddev(static_cast<FloatType>(stddev_)),
int_scale(int_scale_) {
int_scale(int_scale_),
exclude_zero(exclude_zero_) {
float_scale_up = FloatType(IntType(1) << int_scale);
float_scale_up += FloatType(0.5) * float_scale_up;
@@ -272,6 +288,18 @@ struct RandomGaussianFunc<complex<Real>> {
result = Element(Real(rnd_r), Real(rnd_i));
}
if (params.exclude_zero >= 0 &&
result.real() == Real(0.0) &&
result.imag() == Real(0.0)) {
if (rnd_r > FloatType(0)) {
rnd_r += FloatType(1);
} else {
rnd_r -= FloatType(1);
}
result = Element(Real(rnd_r), Real(rnd_i));
}
return result;
}
};
@@ -358,6 +386,7 @@ void TensorFillRandomGaussian(
int bits = -1, ///< If non-negative, specifies number of fractional bits that
/// are not truncated to zero. Permits reducing precision of
/// data.
int exclude_zero = -1, ///< If non-negative, excludes zeros from tensor init
cudaStream_t stream = nullptr) {
using RandomFunc = detail::RandomGaussianFunc<Element>;
@@ -366,7 +395,7 @@ void TensorFillRandomGaussian(
TensorForEach<Func, Layout::kRank, Params>(
view.extent(),
Params(view, typename RandomFunc::Params(seed, mean, stddev, bits)),
Params(view, typename RandomFunc::Params(seed, mean, stddev, bits, exclude_zero)),
/*grid_size*/0, /*block_size*/0,
stream
);
@@ -399,7 +428,7 @@ void BlockFillRandomGaussian(
namespace detail {
/// Computes a random Gaussian distribution
/// Computes a random uniform distribution
template <typename Element> ///< Element type
struct RandomUniformFunc {
@@ -424,8 +453,10 @@ struct RandomUniformFunc {
FloatType range;
FloatType max;
int int_scale;
double pnan;
FloatType float_scale_up;
FloatType float_scale_down;
int exclude_zero; ///< If non-negative, excludes zeros
/// Default ctor
CUTLASS_HOST_DEVICE
@@ -440,15 +471,25 @@ struct RandomUniformFunc {
uint64_t seed_ = 0,
Element max_ = 1,
Element min = 0,
int int_scale_ = -1
int int_scale_ = -1,
double pnan_ = 0,
int exclude_zero_ = -1
):
seed(seed_),
range(static_cast<FloatType>(max_) - static_cast<FloatType>(min)),
max(static_cast<FloatType>(max_)),
int_scale(int_scale_) {
int_scale(int_scale_),
pnan(pnan_),
exclude_zero(exclude_zero_) {
float_scale_up = FloatType(IntType(2) << int_scale); // scale up to clamp low order bits
float_scale_down = FloatType(1) / FloatType(IntType(2) << int_scale);
// Handle cases where min = 0 or max = 0 for excluding zeros
if (exclude_zero >= 0) {
range = (min == Element(0)) ? range - FloatType(1): range;
max = (max_ == Element(0)) ? max - FloatType(1): max;
}
}
};
@@ -479,6 +520,13 @@ struct RandomUniformFunc {
CUTLASS_DEVICE
Element operator()() {
// Draw random float in [0.0, 1.0] to determine if element should be NaN.
if constexpr (std::numeric_limits<Element>::has_quiet_NaN) {
if (params.pnan > 0 && (curand_uniform(&rng_state) < (params.pnan))) {
return Element(NAN);
}
}
FloatType rnd = random_uniform_float<FloatType>(&rng_state);
rnd = params.max - params.range * rnd;
@@ -494,6 +542,15 @@ struct RandomUniformFunc {
result = Element(rnd);
}
if (params.exclude_zero >=0 && result == Element(0.0)) {
if (rnd > FloatType(0)) {
rnd = std::min(params.max, rnd + FloatType(1));
} else {
rnd = std::max((params.max - params.range), rnd - FloatType(1));
}
result = Element(rnd);
}
return result;
}
};
@@ -525,8 +582,10 @@ struct RandomUniformFunc<complex<Real>> {
FloatType range;
FloatType min;
int int_scale;
double pnan;
FloatType float_scale_up;
FloatType float_scale_down;
int exclude_zero; ///< If non-negative, excludes zeros
/// Default ctor
CUTLASS_HOST_DEVICE
@@ -541,16 +600,26 @@ struct RandomUniformFunc<complex<Real>> {
uint64_t seed_ = 0,
FloatType max = 1,
FloatType min_ = 0,
int int_scale_ = -1
int int_scale_ = -1,
double pnan_ = 0,
int exclude_zero_ = -1
):
seed(seed_),
range(static_cast<FloatType>(max - min_)),
min(static_cast<FloatType>(min_)),
int_scale(int_scale_) {
int_scale(int_scale_),
pnan(pnan_),
exclude_zero(exclude_zero_) {
float_scale_up = FloatType(IntType(1) << int_scale);
float_scale_up += FloatType(0.5) * float_scale_up;
float_scale_down = FloatType(1) / FloatType(IntType(1) << int_scale);
// Handle cases where min = 0 or max = 0 for excluding zeros
if (exclude_zero >= 0) {
min = (min == FloatType(0)) ? min + FloatType(1): min;
range = (max == FloatType(0)) ? range - FloatType(1): range;
}
}
};
@@ -581,6 +650,13 @@ struct RandomUniformFunc<complex<Real>> {
CUTLASS_DEVICE
Element operator()() {
// Draw random float in [0.0, 1.0] to determine if element should be NaN.
if constexpr (std::numeric_limits<Element>::has_quiet_NaN) {
if (params.pnan > 0 && (curand_uniform(&rng_state) < (params.pnan))) {
return Element(Real(NAN), Real(NAN));
}
}
FloatType rnd_r = random_uniform_float<FloatType>(&rng_state);
FloatType rnd_i = random_uniform_float<FloatType>(&rng_state);
@@ -604,11 +680,23 @@ struct RandomUniformFunc<complex<Real>> {
result = Element(Real(rnd_r), Real(rnd_i));
}
if (params.exclude_zero >= 0 &&
result.real() == Real(0.0) &&
result.imag() == Real(0.0)) {
if (rnd_r > FloatType(0)) {
rnd_r = std::min(params.min + params.range, rnd_r + FloatType(1));
} else {
rnd_r = std::max((params.min), rnd_r - FloatType(1));
}
result = Element(Real(rnd_r), Real(rnd_i));
}
return result;
}
};
/// Computes a random Gaussian distribution
/// Computes a random uniform distribution
template <
typename Element, ///< Element type
typename Layout> ///< Layout function
@@ -693,13 +781,15 @@ void TensorFillRandomUniform(
int bits = -1, ///< If non-negative, specifies number of fractional bits that
/// are not truncated to zero. Permits reducing precision of
/// data.
double pnan = 0, ///< Percentage of NaN elements.
int exclude_zero = -1, ///< If non-negative, excludes zeros from tensor init
cudaStream_t stream = nullptr) {
using RandomFunc = detail::RandomUniformFunc<Element>;
using Func = detail::TensorFillRandomUniformFunc<Element, Layout>;
using Params = typename Func::Params;
typename RandomFunc::Params random(seed, max, min, bits);
typename RandomFunc::Params random(seed, max, min, bits, pnan, exclude_zero);
TensorForEach<Func, Layout::kRank, Params>(
view.extent(),
@@ -722,11 +812,12 @@ void BlockFillRandomUniform(
int bits = -1, ///< If non-negative, specifies number of fractional bits that
/// are not truncated to zero. Permits reducing precision of
/// data.
double pnan = 0, ///< Percentage of NaN elements.
cudaStream_t stream = nullptr) {
using RandomFunc = detail::RandomUniformFunc<Element>;
typename RandomFunc::Params params(seed, max, min, bits);
typename RandomFunc::Params params(seed, max, min, bits, pnan);
BlockForEach<Element, RandomFunc>(ptr, capacity, params, /*grid_size*/0, /*block_size*/0, stream);
}
@@ -1672,7 +1763,11 @@ void TensorFillRandom(
TensorView<Element, Layout> view, ///< destination tensor
uint64_t seed,
Distribution dist,
cudaStream_t stream = nullptr) {
cudaStream_t stream = nullptr,
int exclude_zero = -1 ///< If non-negative, excludes 0.
/// Note that setting this flag will result in more 1's,
/// as we use a simple mechanism to replace 0's by adding/subtracting 1's.
) {
using Real = typename RealType<Element>::Type;
@@ -1683,6 +1778,7 @@ void TensorFillRandom(
static_cast<Real>(dist.gaussian.mean),
static_cast<Real>(dist.gaussian.stddev),
dist.int_scale,
exclude_zero,
stream);
} else if (dist.kind == Distribution::Uniform) {
TensorFillRandomUniform<Element, Layout>(
@@ -1691,6 +1787,8 @@ void TensorFillRandom(
static_cast<Real>(dist.uniform.max),
static_cast<Real>(dist.uniform.min),
dist.int_scale,
dist.uniform.pnan,
exclude_zero,
stream);
}
}
@@ -1753,6 +1851,7 @@ void BlockFillRandom(
static_cast<Real>(dist.uniform.max),
static_cast<Real>(dist.uniform.min),
dist.int_scale,
dist.uniform.pnan,
stream);
}
}
tools/util/include/cutlass/util/reference/host/conv.hpp
+2 -1
View File
@@ -128,7 +128,8 @@ template<
class EpilogueFusionParams
>
struct ConvReferenceImpl {
using ElementAcc = typename EpilogueFusionParams::ElementAcc;
// Hard code accumlulator type to float to avoid data lost in accumulating add.
using ElementAcc = cutlass::platform::conditional_t<cutlass::platform::is_same_v<typename EpilogueFusionParams::ElementAcc, double>, double, float>;
using ElementC = typename EpilogueFusionParams::ElementC;
using ElementOut = typename EpilogueFusionParams::ElementOut;
using ElementScalar = typename EpilogueFusionParams::ElementScalar;
tools/util/include/cutlass/util/reference/host/gett.hpp
+4 -2
View File
@@ -342,7 +342,8 @@ void gett_epilogue(
ElementCompute converted_acc = accumulator_converter(acc[m_b][n_b]);
// per-row alpha
if (raw_pointer_cast(epilogue_params.Valpha.data())) {
converted_alpha = scale_converter(epilogue_params.Valpha(m + m_b));
converted_alpha = scale_converter(epilogue_params.Valpha(m + m_b, n + n_b, l));
converted_alpha = mul(converted_alpha, mul(converted_scale_a, converted_scale_b));
}
ElementCompute output = mul(converted_alpha, converted_acc);
@@ -355,7 +356,8 @@ void gett_epilogue(
ElementCompute converted_src = source_converter(epilogue_params.C(m + m_b, n + n_b, l));
// per-row beta
if (epilogue_params.Vbeta.data()) {
converted_beta = scale_converter(epilogue_params.Vbeta(m + m_b));
converted_beta = scale_converter(epilogue_params.Vbeta(m + m_b, n + n_b, l));
converted_beta = mul(converted_beta, converted_scale_c);
}
output = epilogue_fma(converted_beta, converted_src, output);
}
tools/util/include/cutlass/util/reference/host/tensor_fill.h
+197 -46
View File
@@ -159,6 +159,7 @@ struct RandomGaussianFunc {
int int_scale;
double pi;
double pnz;
bool exclude_zero;
//
// Methods
@@ -168,9 +169,10 @@ struct RandomGaussianFunc {
double mean_ = 0,
double stddev_ = 1,
int int_scale_ = -1,
double pnz_ = 100.0
double pnz_ = 1.0,
bool exclude_zero_ = false
):
seed(seed_), mean(mean_), stddev(stddev_), int_scale(int_scale_), pi(std::acos(-1)), pnz(pnz_) {
seed(seed_), mean(mean_), stddev(stddev_), int_scale(int_scale_), pi(std::acos(-1)), pnz(pnz_), exclude_zero(exclude_zero_) {
std::srand((unsigned)seed);
}
@@ -191,7 +193,7 @@ struct RandomGaussianFunc {
// Sample from the Bernoulli distribution, and use the result to sample from the Gaussian
std::random_device rnd_device;
std::mt19937 bernoulli_rnd(rnd_device());
std::bernoulli_distribution bernoulli_dist(pnz / 100);
std::bernoulli_distribution bernoulli_dist(pnz);
bool bernoulli_result = bernoulli_dist(bernoulli_rnd);
// Sample from the Gaussian distribution for a nonzero element
@@ -208,6 +210,16 @@ struct RandomGaussianFunc {
result = static_cast<Element>(0);
}
// Note that exclude_zero = true will disable the bernoulli_result above by unsetting zeros
if (exclude_zero && result == Element(0)) {
if (rnd > 0) {
rnd += 1;
} else {
rnd -= 1;
}
result = Element(rnd);
}
return result;
}
};
@@ -222,6 +234,7 @@ struct RandomGaussianFunc<complex<Element> > {
int int_scale;
double pi;
double pnz;
bool exclude_zero;
//
// Methods
@@ -231,9 +244,10 @@ struct RandomGaussianFunc<complex<Element> > {
double mean_ = 0,
double stddev_ = 1,
int int_scale_ = -1,
double pnz_ = 100.0
double pnz_ = 1.0,
bool exclude_zero_ = false
):
seed(seed_), mean(mean_), stddev(stddev_), int_scale(int_scale_), pi(std::acos(-1)), pnz(pnz_) {
seed(seed_), mean(mean_), stddev(stddev_), int_scale(int_scale_), pi(std::acos(-1)), pnz(pnz_), exclude_zero(exclude_zero_) {
std::srand((unsigned)seed);
}
@@ -249,7 +263,7 @@ struct RandomGaussianFunc<complex<Element> > {
// Sample from the Bernoulli distribution, and use the result to sample from the Gaussian
std::random_device rnd_device;
std::mt19937 bernoulli_rnd(rnd_device());
std::bernoulli_distribution bernoulli_dist(pnz / 100);
std::bernoulli_distribution bernoulli_dist(pnz);
bool bernoulli_result = bernoulli_dist(bernoulli_rnd);
// Sample from the Gaussian distribution for a nonzero element
@@ -270,6 +284,19 @@ struct RandomGaussianFunc<complex<Element> > {
reals[1] = from_real<Element>(0);
}
// Note that this will invalidate the above else statement because it unsets zero elements
if (exclude_zero &&
reals[0] == from_real<Element>(0.0) &&
reals[1] == from_real<Element>(0.0)) {
if (rnd[0] > 0.0) {
rnd[0] += 1.0;
} else {
rnd[0] -= 1.0;
}
reals[0] = from_real<Element>(rnd[0]);
}
return complex<Element>(reals[0], reals[1]);
}
};
@@ -284,6 +311,7 @@ struct RandomGaussianFunc<Quaternion<Element> > {
int int_scale;
double pi;
double pnz;
bool exclude_zero;
//
// Methods
@@ -293,9 +321,10 @@ struct RandomGaussianFunc<Quaternion<Element> > {
double mean_ = 0,
double stddev_ = 1,
int int_scale_ = -1,
double pnz_ = 100.0
double pnz_ = 1.0,
bool exclude_zero_ = false
):
seed(seed_), mean(mean_), stddev(stddev_), int_scale(int_scale_), pi(std::acos(-1)), pnz(pnz_) {
seed(seed_), mean(mean_), stddev(stddev_), int_scale(int_scale_), pi(std::acos(-1)), pnz(pnz_), exclude_zero(exclude_zero_) {
std::srand((unsigned)seed);
}
@@ -313,7 +342,7 @@ struct RandomGaussianFunc<Quaternion<Element> > {
// Sample from the Bernoulli distribution, and use the result to sample from the Gaussian
std::random_device rnd_device;
std::mt19937 bernoulli_rnd(rnd_device());
std::bernoulli_distribution bernoulli_dist(pnz / 100);
std::bernoulli_distribution bernoulli_dist(pnz);
bool bernoulli_result = bernoulli_dist(bernoulli_rnd);
// Sample from the Gaussian distribution for a nonzero element
@@ -343,6 +372,21 @@ struct RandomGaussianFunc<Quaternion<Element> > {
reals[3] = from_real<Element>(0);
}
// Note that this will invalidate the above else statement because it unsets zero elements
if (exclude_zero &&
reals[0] == from_real<Element>(0) &&
reals[1] == from_real<Element>(0) &&
reals[2] == from_real<Element>(0) &&
reals[3] == from_real<Element>(0)) {
if (rnd1[0] > 0.0) {
rnd1[0] += 1.0;
} else {
rnd1[0] -= 1.0;
}
reals[0] = from_real<Element>(rnd1[0]);
}
return Quaternion<Element>(reals[0], reals[1], reals[2], reals[3]);
}
};
@@ -440,10 +484,11 @@ void TensorFillRandomGaussian(
double mean = 0, ///< Gaussian distribution's mean
double stddev = 1, ///< Gaussian distribution's standard deviation
int bits = -1, ///< If non-negative, specifies number of fractional bits that
double pnz = 100.0) { /// are not truncated to zero. Permits reducing precision of
double pnz = 1.0, /// are not truncated to zero. Permits reducing precision of
/// data.
bool exclude_zero = false) { ///< Exclude zeros from tensor init.
detail::RandomGaussianFunc<Element> random_func(seed, mean, stddev, bits, pnz);
detail::RandomGaussianFunc<Element> random_func(seed, mean, stddev, bits, pnz, exclude_zero);
detail::TensorFillGaussianFunc<Element, Layout> func(
dst,
@@ -466,8 +511,9 @@ void TensorFillRandomGaussian(
double mean = 0, ///< Gaussian distribution's mean
double stddev = 1, ///< Gaussian distribution's standard deviation
int bits = -1, ///< If non-negative, specifies number of fractional bits that
double pnz = 100.0) { /// are not truncated to zero. Permits reducing precision of
double pnz = 1.0, /// are not truncated to zero. Permits reducing precision of
/// data.
bool exclude_zero = false) { ///< Exclude zeros from tensor init.
TensorFillRandomGaussian(dst.view_real(), seed, mean, stddev, bits, pnz);
TensorFillRandomGaussian(dst.view_imag(), ~seed, mean, stddev, bits, pnz);
@@ -485,7 +531,7 @@ void TensorFillSymmetricRandomGaussian(
double mean = 0, ///< Gaussian distribution's mean
double stddev = 1, ///< Gaussian distribution's standard deviation
int bits = -1, ///< If non-negative, specifies number of fractional bits that
double pnz = 100.0) { /// are not truncated to zero. Permits reducing precision of
double pnz = 1.0) { /// are not truncated to zero. Permits reducing precision of
/// data.
detail::RandomGaussianFunc<Element> random_func(seed, mean, stddev, bits, pnz);
@@ -515,7 +561,7 @@ void BlockFillRandomGaussian(
double mean = 0, ///< Gaussian distribution's mean
double stddev = 1, ///< Gaussian distribution's standard deviation
int bits = -1, ///< If non-negative, specifies number of fractional bits that
double pnz = 100.0) { /// are not truncated to zero. Permits reducing precision of
double pnz = 1.0) { /// are not truncated to zero. Permits reducing precision of
/// data.
@@ -542,23 +588,47 @@ struct RandomUniformFunc {
double min;
int int_scale;
//
// Methods
//
double pnan;
private:
using engine_type = std::mt19937;
public:
engine_type bernoulli_rnd;
std::bernoulli_distribution bernoulli_dist;
bool exclude_zero;
RandomUniformFunc(
uint64_t seed_ = 0,
double max = 1,
double min_ = 0,
int int_scale_ = -1
int int_scale_ = -1,
double pnan_ = 0,
bool exclude_zero_ = false
):
seed(seed_), range(max - min_), min(min_), int_scale(int_scale_) {
seed(seed_), range(max - min_), min(min_), int_scale(int_scale_), pnan(pnan_)
, bernoulli_rnd{static_cast<engine_type::result_type>(seed_)}
, bernoulli_dist(pnan_)
, exclude_zero(exclude_zero_)
{
std::srand((unsigned)seed);
}
// Handle cases where min = 0 or max = 0 for excluding zeros
if (exclude_zero) {
min = (min == 0.0) ? min + 1: min;
range = (max == 0.0) ? range - 1: range;
}
}
/// Compute random value and update RNG state
Element operator()() const {
Element operator()() {
// Sample from NaN distribution.
if constexpr (std::numeric_limits<Element>::has_quiet_NaN) {
if (pnan > 0 && bernoulli_dist(bernoulli_rnd)) {
return Element(NAN);
}
}
double rnd = double(std::rand()) / double(RAND_MAX);
@@ -575,6 +645,15 @@ struct RandomUniformFunc {
result = static_cast<Element>(Real(rnd));
}
if (exclude_zero && result == Element(0)) {
if (rnd > 0.0) {
rnd = std::min(min + range, rnd + 1.0);
} else {
rnd = std::max(min, rnd - 1.0);
}
result = static_cast<Element>(Real(rnd));
}
return result;
}
};
@@ -590,6 +669,15 @@ struct RandomUniformFunc<complex<Element> > {
double min;
int int_scale;
double pnan;
private:
using engine_type = std::mt19937;
public:
engine_type bernoulli_rnd;
std::bernoulli_distribution bernoulli_dist;
bool exclude_zero;
//
// Methods
//
@@ -598,15 +686,33 @@ struct RandomUniformFunc<complex<Element> > {
uint64_t seed_ = 0,
double max = 1,
double min_ = 0,
int int_scale_ = -1
int int_scale_ = -1,
double pnan_ = 0,
bool exclude_zero_ = false
):
seed(seed_), range(max - min_), min(min_), int_scale(int_scale_) {
seed(seed_), range(max - min_), min(min_), int_scale(int_scale_), pnan(pnan_)
, bernoulli_rnd{static_cast<engine_type::result_type>(seed_)}
, bernoulli_dist(pnan_)
, exclude_zero(exclude_zero_) {
std::srand((unsigned)seed);
}
// Handle cases where min = 0 or max = 0 for excluding zeros
if (exclude_zero) {
min = (min == 0.0) ? min + 1: min;
range = (max == 0.0) ? range - 1: range;
}
}
/// Compute random value and update RNG state
complex<Element> operator()() const {
complex<Element> operator()() {
// Sample from NaN distribution.
if constexpr (std::numeric_limits<Element>::has_quiet_NaN) {
if (pnan > 0 && bernoulli_dist(bernoulli_rnd)) {
return Element(NAN);
}
}
Element reals[2];
@@ -625,6 +731,19 @@ struct RandomUniformFunc<complex<Element> > {
else {
reals[i] = from_real<Element>(Real(rnd));
}
if (exclude_zero &&
i == 0 &&
reals[0] == from_real<Element>(0.0)) {
if (rnd > 0.0) {
rnd = std::min(min + range, rnd + 1.0);
} else {
rnd = std::max(min, rnd - 1.0);
}
reals[0] = from_real<Element>(Real(rnd));
}
}
return complex<Element>(reals[0], reals[1]);
@@ -642,6 +761,13 @@ struct RandomUniformFunc<Quaternion<Element> > {
double min;
int int_scale;
double pnan;
private:
using engine_type = std::mt19937;
public:
engine_type bernoulli_rnd;
std::bernoulli_distribution bernoulli_dist;
//
// Methods
//
@@ -650,15 +776,26 @@ struct RandomUniformFunc<Quaternion<Element> > {
uint64_t seed_ = 0,
double max = 1,
double min_ = 0,
int int_scale_ = -1
int int_scale_ = -1,
double pnan_ = 0
):
seed(seed_), range(max - min_), min(min_), int_scale(int_scale_) {
std::srand((unsigned)seed);
}
seed(seed_), range(max - min_), min(min_), int_scale(int_scale_), pnan(pnan_),
bernoulli_rnd{static_cast<engine_type::result_type>(seed_)},
bernoulli_dist(pnan_)
{
std::srand((unsigned)seed);
}
/// Compute random value and update RNG state
Quaternion<Element> operator()() const {
Quaternion<Element> operator()() {
// Sample from NaN distribution.
if constexpr (std::numeric_limits<Element>::has_quiet_NaN) {
if (pnan > 0 && bernoulli_dist(bernoulli_rnd)) {
return Element(NAN);
}
}
Element reals[4];
@@ -712,7 +849,7 @@ struct TensorFillRandomUniformFunc {
}
/// Compute random value and update RNG state
void operator()(Coord<Layout::kRank> const &coord) const {
void operator()(Coord<Layout::kRank> const &coord) {
view.at(coord) = func();
}
@@ -749,7 +886,7 @@ struct TensorFillSymmetricRandomUniformFunc {
}
/// Compute random value and update RNG state
void operator()(Coord<Layout::kRank> const &coord) const {
void operator()(Coord<Layout::kRank> const &coord) {
// Fill half of matrix based on FillMode
if (Layout::kRank == 2 &&
fill_mode == cutlass::FillMode::kLower &&
@@ -796,7 +933,7 @@ struct TensorFillPadDiagonalRandomUniformFunc {
}
/// Compute random value and update RNG state
void operator()(Coord<Layout::kRank> const &coord) const {
void operator()(Coord<Layout::kRank> const &coord) {
// Fill half of matrix based on FillMode
if (Layout::kRank == 2 &&
(fill_mode == cutlass::FillMode::kLower) &&
@@ -825,10 +962,12 @@ void TensorFillRandomUniform(
uint64_t seed, ///< seed for RNG
double max = 1, ///< upper bound of distribution
double min = 0, ///< lower bound for distribution
int bits = -1) { ///< If non-negative, specifies number of fractional bits that
int bits = -1, ///< If non-negative, specifies number of fractional bits that
/// are not truncated to zero. Permits reducing precision of
/// data.
detail::RandomUniformFunc<Element> random_func(seed, max, min, bits);
/// data.
double pnan = 0, ///< Percentage of NaN elements.
bool exclude_zero = false) { ///< Exclude zero from tensor init
detail::RandomUniformFunc<Element> random_func(seed, max, min, bits, pnan, exclude_zero);
detail::TensorFillRandomUniformFunc<Element, Layout> func(
dst,
@@ -850,12 +989,14 @@ void TensorFillRandomUniform(
uint64_t seed, ///< seed for RNG
double max = 1, ///< upper bound of distribution
double min = 0, ///< lower bound for distribution
int bits = -1) { ///< If non-negative, specifies number of fractional bits that
int bits = -1, ///< If non-negative, specifies number of fractional bits that
/// are not truncated to zero. Permits reducing precision of
/// data.
double pnan = 0, ///< Percentage of NaN elements.
bool exclude_zero = false) { ///< Exclude zero from tensor init
TensorFillRandomUniform(dst.view_real(), seed, max, min, bits);
TensorFillRandomUniform(dst.view_imag(), ~seed, max, min, bits);
TensorFillRandomUniform(dst.view_real(), seed, max, min, bits, pnan, exclude_zero);
TensorFillRandomUniform(dst.view_imag(), ~seed, max, min, bits, pnan, exclude_zero);
}
@@ -972,10 +1113,11 @@ void BlockFillRandomUniform(
uint64_t seed, ///< seed for RNG
double max = 1, ///< upper bound of distribution
double min = 0, ///< lower bound for distribution
int bits = -1) { ///< If non-negative, specifies number of fractional bits that
int bits = -1, ///< If non-negative, specifies number of fractional bits that
/// are not truncated to zero. Permits reducing precision of
/// data.
detail::RandomUniformFunc<Element> random_func(seed, max, min, bits);
/// data.
double pnan = 0) { ///< Percentage of NaN elements.
detail::RandomUniformFunc<Element> random_func(seed, max, min, bits, pnan);
for (size_t i = 0; i < capacity; ++i) {
ReferenceFactory<Element>::get(ptr, i) = random_func();
@@ -1259,7 +1401,11 @@ template <
void TensorFillRandom(
TensorView<Element, Layout> view, ///< destination tensor
uint64_t seed,
Distribution dist) {
Distribution dist,
bool exclude_zero = false ///< If true, excludes 0.
/// Note that setting this flag will result in more 1's,
/// as we use a simple mechanism to replace 0's by adding/subtracting 1's.
) {
using Real = typename RealType<Element>::Type;
@@ -1269,14 +1415,18 @@ void TensorFillRandom(
seed,
dist.gaussian.mean,
dist.gaussian.stddev,
dist.int_scale);
dist.int_scale,
dist.gaussian.pnz,
exclude_zero);
} else if (dist.kind == Distribution::Uniform) {
TensorFillRandomUniform(
view,
seed,
dist.uniform.max,
dist.uniform.min,
dist.int_scale);
dist.int_scale,
dist.uniform.pnan,
exclude_zero);
}
}
@@ -1354,7 +1504,8 @@ void BlockFillRandom(
seed,
dist.uniform.max,
dist.uniform.min,
dist.int_scale);
dist.int_scale,
dist.uniform.pnan);
}
}