leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Softmax (#546)

Browse Source 
* add test layernorm g-mem version

* Delete include/configure directory

* Delete examples/test_layernorm directory

* Update gemm_with_softmax.h

* Update gemm_softmax.cu

* Update linear_combination.h

* Update fast_math.h

* remove redundant vars

Co-authored-by: yujia.zhai <yujia.zhai@bytedance.com>
Co-authored-by: yuzhai <yuzhai@nvidia.com>
This commit is contained in:
Yujia Zhai
2022-07-01 22:19:18 -07:00
committed by GitHub
parent e45e773436
commit 04a9777b87
4 changed files with 428 additions and 376 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
include/cutlass/epilogue/thread/linear_combination.h
View File

@@ -71,7 +71,7 @@ public:
using ElementCompute = ElementCompute_;
static int const kCount = Count;
static const ScaleType::Kind kScale = Scale;
using FragmentOutput = Array<ElementOutput, kCount>;
using FragmentAccumulator = Array<ElementAccumulator, kCount>;
using ComputeFragment = Array<ElementCompute, kCount>;

7
include/cutlass/fast_math.h
View File

@@ -718,11 +718,11 @@ double fast_exp(double x) {
}
CUTLASS_HOST_DEVICE
float fast_exp(half_t x) {
half_t fast_exp(half_t x) {
#if defined(__CUDA_ARCH__) && (__CUDACC_VER_MAJOR__ >= 10) && (__CUDA_ARCH__ >= 750)
return ::hexp(x.to_half());
return (half_t)(::hexp(x.to_half()));
#else
return fast_exp(float(x));
return (half_t)(fast_exp(float(x)));
#endif
}
@@ -908,4 +908,3 @@ T absolute_value(T x) {
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////