leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 26 Packages Projects Releases Wiki Activity

3.6.0 update (#2005)

Browse Source 
* 3.6.0 update

* doc and swap stuff

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
Yujia Zhai
2024-12-25 01:34:40 -05:00
committed by GitHub
co-authored by yuzhai Haicheng Wu
parent e1cd8c7866
commit 3d261a5974
258 changed files with 10863 additions and 3883 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/util/include/cutlass/util/device_dump.h
+1 -1
View File
@@ -31,7 +31,7 @@
#pragma once
#include <stdio.h>
#include <cstdio>
#include "cutlass/cutlass.h"
/**
tools/util/include/cutlass/util/device_groupnorm.h
+1 -1
View File
@@ -42,7 +42,7 @@
#include "cutlass/tensor_coord.h"
#include "cutlass/tensor_ref.h"
#include "device_utils.h"
#include <float.h>
#include <cfloat>
namespace cutlass {
tools/util/include/cutlass/util/device_layernorm.h
+1 -1
View File
@@ -42,7 +42,7 @@
#include "cutlass/tensor_coord.h"
#include "cutlass/tensor_ref.h"
#include "device_utils.h"
#include <float.h>
#include <cfloat>
namespace cutlass {
tools/util/include/cutlass/util/device_nhwc_pooling.h
+1 -1
View File
@@ -42,7 +42,7 @@
#include "cutlass/tensor_coord.h"
#include "cutlass/tensor_ref.h"
#include "device_utils.h"
#include <float.h>
#include <cfloat>
namespace cutlass {
tools/util/include/cutlass/util/device_rmsnorm.h
+3 -3
View File
@@ -37,7 +37,7 @@
#include "cutlass/tensor_coord.h"
#include "cutlass/tensor_ref.h"
#include "cutlass/util/device_utils.h"
#include <float.h>
#include <cfloat>
namespace cutlass {
@@ -165,12 +165,12 @@ void rmsnorm(cutlass::MatrixCoord tensor_size,
dim3 grid(m);
if (n % 8 == 0 && std::is_same<T, cutlass::half_t>::value) {
dim3 block(min(1024, (n / 8 + 31) / 32 * 32));
dim3 block(cutlass::platform::min(1024, (n / 8 + 31) / 32 * 32));
rmsnorm_twoPassAlgo_e8<<<grid, block, 0, stream>>>(
(float4 *)output, (const float4 *)input, (const float4 *)weight, m, n, epsilon);
} else {
dim3 block(min(1024, ((n + 31)/32 + 31)/32*32));
dim3 block(cutlass::platform::min(1024, ((n + 31)/32 + 31)/32*32));
rmsnorm_twoPassAlgo_e1<<<grid, block, 0, stream>>>(
output, input, weight, m, n, epsilon);
tools/util/include/cutlass/util/device_utils.h
+1 -1
View File
@@ -36,7 +36,7 @@
#pragma once
#include <cuda_fp16.h>
#include <float.h>
#include <cfloat>
#define FINAL_MASK 0xffffffff
struct half4 {
tools/util/include/cutlass/util/distribution.h
+3
View File
@@ -100,6 +100,9 @@ struct Distribution {
gaussian.mean = _mean;
gaussian.stddev = _stddev;
gaussian.pnz = _pnz;
gaussian.pnzA = _pnz;
gaussian.pnzB = _pnz;
gaussian.pnzC = _pnz;
int_scale = _int_scale;
return *this;
}
tools/util/include/cutlass/util/reference/device/tensor_fill.h
+12 -14
View File
@@ -138,8 +138,8 @@ struct RandomGaussianFunc {
int_scale(int_scale_),
exclude_zero(exclude_zero_) {
float_scale_up = FloatType(IntType(2) << int_scale); // scale up to clamp low order bits
float_scale_down = FloatType(1) / FloatType(IntType(2) << int_scale);
float_scale_up = FloatType(IntType(1) << int_scale); // scale up to clamp low order bits
float_scale_down = FloatType(1) / FloatType(IntType(1) << int_scale);
}
};
@@ -175,8 +175,8 @@ struct RandomGaussianFunc {
Element result;
if (params.int_scale >= 0) {
rnd = FloatType(IntType(std::llround(rnd * params.float_scale_up)));
result = Element(IntType(rnd * params.float_scale_down));
rnd = FloatType(std::llround(rnd * params.float_scale_up));
result = Element(rnd * params.float_scale_down);
}
else {
result = Element(rnd);
@@ -237,7 +237,6 @@ struct RandomGaussianFunc<complex<Real>> {
exclude_zero(exclude_zero_) {
float_scale_up = FloatType(IntType(1) << int_scale);
float_scale_up += FloatType(0.5) * float_scale_up;
float_scale_down = FloatType(1) / FloatType(IntType(1) << int_scale);
}
};
@@ -276,8 +275,8 @@ struct RandomGaussianFunc<complex<Real>> {
Element result;
if (params.int_scale >= 0) {
rnd_r = FloatType(IntType(rnd_r * params.float_scale_up));
rnd_i = FloatType(IntType(rnd_i * params.float_scale_down));
rnd_r = FloatType(std::llround(rnd_r * params.float_scale_up));
rnd_i = FloatType(std::llround(rnd_i * params.float_scale_up));
result = {
Real(rnd_r * params.float_scale_down),
@@ -482,8 +481,8 @@ struct RandomUniformFunc {
pnan(pnan_),
exclude_zero(exclude_zero_) {
float_scale_up = FloatType(IntType(2) << int_scale); // scale up to clamp low order bits
float_scale_down = FloatType(1) / FloatType(IntType(2) << int_scale);
float_scale_up = FloatType(IntType(1) << int_scale); // scale up to clamp low order bits
float_scale_down = FloatType(1) / FloatType(IntType(1) << int_scale);
// Handle cases where min = 0 or max = 0 for excluding zeros
if (exclude_zero >= 0) {
@@ -535,8 +534,8 @@ struct RandomUniformFunc {
Element result;
if (params.int_scale >= 0) {
rnd = FloatType(IntType(std::llround(rnd * params.float_scale_up)));
result = Element(IntType(rnd * params.float_scale_down));
rnd = FloatType(std::llround(rnd * params.float_scale_up));
result = Element(rnd * params.float_scale_down);
}
else {
result = Element(rnd);
@@ -612,7 +611,6 @@ struct RandomUniformFunc<complex<Real>> {
exclude_zero(exclude_zero_) {
float_scale_up = FloatType(IntType(1) << int_scale);
float_scale_up += FloatType(0.5) * float_scale_up;
float_scale_down = FloatType(1) / FloatType(IntType(1) << int_scale);
// Handle cases where min = 0 or max = 0 for excluding zeros
@@ -668,8 +666,8 @@ struct RandomUniformFunc<complex<Real>> {
Element result;
if (params.int_scale >= 0) {
rnd_r = FloatType(IntType(rnd_r * params.float_scale_up));
rnd_i = FloatType(IntType(rnd_i * params.float_scale_up));
rnd_r = FloatType(std::llround(rnd_r * params.float_scale_up));
rnd_i = FloatType(std::llround(rnd_i * params.float_scale_up));
result = {
Real(rnd_r * params.float_scale_down),
tools/util/include/cutlass/util/reference/host/gett.hpp
-1
View File
@@ -281,7 +281,6 @@ void gett_epilogue(
cute::is_same_v<ElementAux, cutlass::uint1b_t>;
constexpr bool IsClamp =
cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::Clamp<ElementCompute>>;
constexpr bool IsBackpropFusion =
cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::dGELU<ElementCompute>> or
cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::dReLU<ElementCompute>>;
tools/util/include/cutlass/util/reference/host/rank_2k_complex.h
+1 -1
View File
@@ -41,7 +41,7 @@
#include "cutlass/numeric_conversion.h"
#include "cutlass/tensor_view.h"
#include "cutlass/gemm/gemm.h"
#include <assert.h>
#include <cassert>
namespace cutlass {
namespace reference {
tools/util/include/cutlass/util/reference/host/rank_k_complex.h
+1 -1
View File
@@ -41,7 +41,7 @@
#include "cutlass/numeric_conversion.h"
#include "cutlass/tensor_view.h"
#include "cutlass/gemm/gemm.h"
#include <assert.h>
#include <cassert>
namespace cutlass {
namespace reference {
tools/util/include/cutlass/util/reference/host/symm_complex.h
+1 -1
View File
@@ -41,7 +41,7 @@
#include "cutlass/numeric_conversion.h"
#include "cutlass/tensor_view.h"
#include "cutlass/gemm/gemm.h"
#include <assert.h>
#include <cassert>
namespace cutlass {
namespace reference {
tools/util/include/cutlass/util/reference/host/tensor_fill.h
+8 -8
View File
@@ -269,8 +269,8 @@ struct RandomGaussianFunc<complex<Element> > {
// Sample from the Gaussian distribution for a nonzero element
if (bernoulli_result) {
if (int_scale >= 0) {
rnd[0] = double(int(rnd[0] * double(1 << int_scale)));
rnd[1] = double(int(rnd[1] * double(1 << int_scale)));
rnd[0] = double(std::llround(rnd[0] * double(1 << int_scale)));
rnd[1] = double(std::llround(rnd[1] * double(1 << int_scale)));
reals[0] = from_real<Element>(rnd[0] / double(1 << int_scale));
reals[1] = from_real<Element>(rnd[1] / double(1 << int_scale));
}
@@ -348,10 +348,10 @@ struct RandomGaussianFunc<Quaternion<Element> > {
// Sample from the Gaussian distribution for a nonzero element
if (bernoulli_result) {
if (int_scale >= 0) {
rnd1[0] = double(int(rnd1[0] * double(1 << int_scale)));
rnd1[1] = double(int(rnd1[1] * double(1 << int_scale)));
rnd2[0] = double(int(rnd2[0] * double(1 << int_scale)));
rnd2[1] = double(int(rnd2[1] * double(1 << int_scale)));
rnd1[0] = double(std::llround(rnd1[0] * double(1 << int_scale)));
rnd1[1] = double(std::llround(rnd1[1] * double(1 << int_scale)));
rnd2[0] = double(std::llround(rnd2[0] * double(1 << int_scale)));
rnd2[1] = double(std::llround(rnd2[1] * double(1 << int_scale)));
reals[0] = from_real<Element>(rnd1[0] / double(1 << int_scale));
reals[1] = from_real<Element>(rnd1[1] / double(1 << int_scale));
@@ -725,7 +725,7 @@ public:
// testing
if (int_scale >= 0) {
rnd = double(int(rnd * double(1 << int_scale)));
rnd = double(std::llround(rnd * double(1 << int_scale)));
reals[i] = from_real<Element>(Real(rnd / double(1 << int_scale)));
}
else {
@@ -808,7 +808,7 @@ public:
// testing
if (int_scale >= 0) {
rnd = double(int(rnd * double(1 << int_scale)));
rnd = double(std::llround(rnd * double(1 << int_scale)));
reals[i] = from_real<Element>(Real(rnd / double(1 << int_scale)));
}
else {
tools/util/include/cutlass/util/type_traits.h
+1 -1
View File
@@ -36,7 +36,7 @@
#include <cublas_v2.h>
#include <cuda_fp16.h>
#include <stdint.h>
#include <cstdint>
#include "cutlass/numeric_types.h"
#include "cutlass/complex.h"