CUTLASS 3.5.0 (#1411)
This commit is contained in:
Vijay Thakkar
@@ -63,6 +63,9 @@ struct ArithmeticTuple : tuple<T...>
|
||||
template <class... T>
|
||||
struct is_tuple<ArithmeticTuple<T...>> : true_type {};
|
||||
|
||||
template <class... Ts>
|
||||
struct is_flat<ArithmeticTuple<Ts...>> : is_flat<tuple<Ts...>> {};
|
||||
|
||||
template <class... T>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
@@ -108,16 +111,45 @@ template <class... T, class... U>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
operator+(ArithmeticTuple<T...> const& t, tuple<U...> const& u) {
|
||||
constexpr int R = cute::max(int(sizeof...(T)), int(sizeof...(U)));
|
||||
return transform_apply(append<R>(t,Int<0>{}), append<R>(u,Int<0>{}), plus{}, [](auto const&... a){ return make_arithmetic_tuple(a...); });
|
||||
return t + ArithmeticTuple<U...>(u);
|
||||
}
|
||||
|
||||
template <class... T, class... U>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
operator+(tuple<T...> const& t, ArithmeticTuple<U...> const& u) {
|
||||
return ArithmeticTuple<T...>(t) + u;
|
||||
}
|
||||
|
||||
// Subtraction
|
||||
template <class... T, class... U>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
operator-(ArithmeticTuple<T...> const& t, ArithmeticTuple<U...> const& u) {
|
||||
constexpr int R = cute::max(int(sizeof...(T)), int(sizeof...(U)));
|
||||
return transform_apply(append<R>(t,Int<0>{}), append<R>(u,Int<0>{}), plus{}, [](auto const&... a){ return make_arithmetic_tuple(a...); });
|
||||
return transform_apply(append<R>(t,Int<0>{}), append<R>(u,Int<0>{}), minus{}, [](auto const&... a){ return make_arithmetic_tuple(a...); });
|
||||
}
|
||||
|
||||
template <class... T, class... U>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
operator-(ArithmeticTuple<T...> const& t, tuple<U...> const& u) {
|
||||
return t - ArithmeticTuple<U...>(u);
|
||||
}
|
||||
|
||||
template <class... T, class... U>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
operator-(tuple<T...> const& t, ArithmeticTuple<U...> const& u) {
|
||||
return ArithmeticTuple<T...>(t) - u;
|
||||
}
|
||||
|
||||
// Negation
|
||||
template <class... T>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
operator-(ArithmeticTuple<T...> const& t) {
|
||||
return transform_apply(t, negate{}, [](auto const&... a){ return make_arithmetic_tuple(a...); });
|
||||
}
|
||||
|
||||
//
|
||||
@@ -128,7 +160,7 @@ template <auto t, class... U>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
ArithmeticTuple<U...> const&
|
||||
operator+(C<t>, ArithmeticTuple<U...> const& u) {
|
||||
static_assert(t == 0, "Artihmetic tuple op+ error!");
|
||||
static_assert(t == 0, "Arithmetic tuple op+ error!");
|
||||
return u;
|
||||
}
|
||||
|
||||
@@ -136,7 +168,23 @@ template <class... T, auto u>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
ArithmeticTuple<T...> const&
|
||||
operator+(ArithmeticTuple<T...> const& t, C<u>) {
|
||||
static_assert(u == 0, "Artihmetic tuple op+ error!");
|
||||
static_assert(u == 0, "Arithmetic tuple op+ error!");
|
||||
return t;
|
||||
}
|
||||
|
||||
template <auto t, class... U>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
ArithmeticTuple<U...> const&
|
||||
operator-(C<t>, ArithmeticTuple<U...> const& u) {
|
||||
static_assert(t == 0, "Arithmetic tuple op- error!");
|
||||
return -u;
|
||||
}
|
||||
|
||||
template <class... T, auto u>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
ArithmeticTuple<T...> const&
|
||||
operator-(ArithmeticTuple<T...> const& t, C<u>) {
|
||||
static_assert(u == 0, "Arithmetic tuple op- error!");
|
||||
return t;
|
||||
}
|
||||
|
||||
@@ -531,7 +579,7 @@ namespace std
|
||||
template <class... _Tp>
|
||||
struct tuple_size;
|
||||
|
||||
template<size_t _Ip, class... _Tp>
|
||||
template <size_t _Ip, class... _Tp>
|
||||
struct tuple_element;
|
||||
#endif
|
||||
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#include <cute/config.hpp>
|
||||
|
||||
#include <vector_types.h>
|
||||
#include <cutlass/numeric_types.h>
|
||||
|
||||
namespace cute {
|
||||
|
||||
using cutlass::bfloat16_t;
|
||||
|
||||
//
|
||||
// Display utilities
|
||||
//
|
||||
|
||||
#if !defined(__CUDACC_RTC__)
|
||||
CUTE_HOST std::ostream& operator<<(std::ostream& os, bfloat16_t const& v)
|
||||
{
|
||||
return os << float(v);
|
||||
}
|
||||
#endif
|
||||
|
||||
} // end namespace cute
|
||||
@@ -30,9 +30,9 @@
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#include <cute/config.hpp>
|
||||
#include <cute/util/type_traits.hpp>
|
||||
#include <cutlass/complex.h>
|
||||
#include <cute/util/type_traits.hpp>
|
||||
#include <cute/numeric/numeric_types.hpp>
|
||||
|
||||
namespace cute
|
||||
{
|
||||
@@ -44,6 +44,9 @@ using cutlass::real;
|
||||
using cutlass::imag;
|
||||
using cutlass::conj;
|
||||
|
||||
template <class T>
|
||||
static constexpr auto is_complex_v = is_complex<T>::value;
|
||||
|
||||
/// Fused multiply-add for complex numbers
|
||||
template <class T>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#include <cute/config.hpp>
|
||||
#include <vector_types.h>
|
||||
#include <cutlass/numeric_types.h>
|
||||
|
||||
namespace cute {
|
||||
|
||||
using cutlass::half_t;
|
||||
|
||||
} // end namespace cute
|
||||
@@ -36,8 +36,7 @@
|
||||
#include <cstdint>
|
||||
#endif
|
||||
|
||||
#include <cute/numeric/integer_subbyte.hpp>
|
||||
#include <cute/numeric/uint128.hpp>
|
||||
#include <cutlass/numeric_types.h>
|
||||
|
||||
namespace cute
|
||||
{
|
||||
@@ -46,16 +45,16 @@ namespace cute
|
||||
// Signed integers
|
||||
//
|
||||
|
||||
using int2_t = cute::int2b_t;
|
||||
using int4_t = cute::int4b_t;
|
||||
using int8_t = CUTE_STL_NAMESPACE::int8_t;
|
||||
using int16_t = CUTE_STL_NAMESPACE::int16_t;
|
||||
using int32_t = CUTE_STL_NAMESPACE::int32_t;
|
||||
using int64_t = CUTE_STL_NAMESPACE::int64_t;
|
||||
using int2_t = cutlass::int2b_t;
|
||||
using int4_t = cutlass::int4b_t;
|
||||
using CUTE_STL_NAMESPACE::int8_t;
|
||||
using CUTE_STL_NAMESPACE::int16_t;
|
||||
using CUTE_STL_NAMESPACE::int32_t;
|
||||
using CUTE_STL_NAMESPACE::int64_t;
|
||||
|
||||
template <int N> struct int_bit;
|
||||
template <> struct int_bit< 2> { using type = cute::int2b_t; };
|
||||
template <> struct int_bit< 4> { using type = cute::int4b_t; };
|
||||
template <> struct int_bit< 2> { using type = cutlass::int2b_t; };
|
||||
template <> struct int_bit< 4> { using type = cutlass::int4b_t; };
|
||||
template <> struct int_bit< 8> { using type = int8_t; };
|
||||
template <> struct int_bit< 16> { using type = int16_t; };
|
||||
template <> struct int_bit< 32> { using type = int32_t; };
|
||||
@@ -74,24 +73,24 @@ using int_byte_t = typename int_byte<N>::type;
|
||||
// Unsigned integers
|
||||
//
|
||||
|
||||
using uint1_t = cute::uint1b_t;
|
||||
using uint2_t = cute::uint2b_t;
|
||||
using uint4_t = cute::uint4b_t;
|
||||
using uint8_t = CUTE_STL_NAMESPACE::uint8_t;
|
||||
using uint16_t = CUTE_STL_NAMESPACE::uint16_t;
|
||||
using uint32_t = CUTE_STL_NAMESPACE::uint32_t;
|
||||
using uint64_t = CUTE_STL_NAMESPACE::uint64_t;
|
||||
using uint128_t = cute::uint128_t;
|
||||
using uint1_t = cutlass::uint1b_t;
|
||||
using uint2_t = cutlass::uint2b_t;
|
||||
using uint4_t = cutlass::uint4b_t;
|
||||
using CUTE_STL_NAMESPACE::uint8_t;
|
||||
using CUTE_STL_NAMESPACE::uint16_t;
|
||||
using CUTE_STL_NAMESPACE::uint32_t;
|
||||
using CUTE_STL_NAMESPACE::uint64_t;
|
||||
using cutlass::uint128_t;
|
||||
|
||||
template <int N> struct uint_bit;
|
||||
template <> struct uint_bit< 1> { using type = cute::uint1b_t; };
|
||||
template <> struct uint_bit< 2> { using type = cute::uint2b_t; };
|
||||
template <> struct uint_bit< 4> { using type = cute::uint4b_t; };
|
||||
template <> struct uint_bit< 1> { using type = cutlass::uint1b_t; };
|
||||
template <> struct uint_bit< 2> { using type = cutlass::uint2b_t; };
|
||||
template <> struct uint_bit< 4> { using type = cutlass::uint4b_t; };
|
||||
template <> struct uint_bit< 8> { using type = uint8_t; };
|
||||
template <> struct uint_bit< 16> { using type = uint16_t; };
|
||||
template <> struct uint_bit< 32> { using type = uint32_t; };
|
||||
template <> struct uint_bit< 64> { using type = uint64_t; };
|
||||
template <> struct uint_bit<128> { using type = cute::uint128_t; };
|
||||
template <> struct uint_bit<128> { using type = cutlass::uint128_t; };
|
||||
|
||||
template <int N>
|
||||
using uint_bit_t = typename uint_bit<N>::type;
|
||||
@@ -102,50 +101,4 @@ using uint_byte = uint_bit<8*N>;
|
||||
template <int N>
|
||||
using uint_byte_t = typename uint_byte<N>::type;
|
||||
|
||||
//
|
||||
// sizeof_bytes
|
||||
//
|
||||
|
||||
template <class T>
|
||||
struct sizeof_bytes {
|
||||
static constexpr size_t value = sizeof(T);
|
||||
};
|
||||
template <class T>
|
||||
static constexpr int sizeof_bytes_v = sizeof_bytes<T>::value;
|
||||
|
||||
//
|
||||
// sizeof_bits
|
||||
//
|
||||
|
||||
template <class T>
|
||||
struct sizeof_bits {
|
||||
static constexpr size_t value = sizeof(T) * 8;
|
||||
};
|
||||
|
||||
template <class T>
|
||||
struct sizeof_bits<T const>: sizeof_bits<T> {};
|
||||
|
||||
template <>
|
||||
struct sizeof_bits<void> {
|
||||
static constexpr size_t value = 0;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct sizeof_bits<bool> {
|
||||
static constexpr size_t value = 1;
|
||||
};
|
||||
|
||||
template <int Bits, bool Signed>
|
||||
struct sizeof_bits<integer_subbyte<Bits,Signed>> {
|
||||
static constexpr size_t value = Bits;
|
||||
};
|
||||
|
||||
template <int Bits, bool Signed>
|
||||
struct sizeof_bits<cutlass::integer_subbyte<Bits,Signed>> {
|
||||
static constexpr size_t value = Bits;
|
||||
};
|
||||
|
||||
template <class T>
|
||||
static constexpr int sizeof_bits_v = sizeof_bits<T>::value;
|
||||
|
||||
} // namespace cute
|
||||
|
||||
@@ -1,235 +0,0 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#if defined(__CUDACC_RTC__)
|
||||
#include <cuda/std/cstdint>
|
||||
#else
|
||||
#include <cstdint>
|
||||
#endif
|
||||
|
||||
#include <cutlass/integer_subbyte.h>
|
||||
|
||||
#include <cute/config.hpp>
|
||||
#include <cute/util/type_traits.hpp>
|
||||
|
||||
namespace cute {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <int Bits, bool Signed = true>
|
||||
struct integer_subbyte
|
||||
{
|
||||
/// Storage type
|
||||
using Storage = uint8_t;
|
||||
|
||||
/// Number of bits
|
||||
static_assert(Bits <= 8*sizeof(Storage), "Require a subbyte of bits in integer_subbyte");
|
||||
|
||||
/// External type
|
||||
using xint_t = typename conditional<Signed, int, unsigned>::type;
|
||||
|
||||
/// Bitmask for truncation from larger integers
|
||||
static constexpr Storage bits_mask_ = Storage((1 << Bits) - 1);
|
||||
/// Bitmask for the sign bit
|
||||
static constexpr Storage sign_mask_ = Storage((Signed ? 1 : 0) << (Bits - 1));
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
Storage storage;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// No operation
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
integer_subbyte() {}
|
||||
|
||||
/// Conversion from integer type
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
integer_subbyte(int value) // NOTE: Sign extension?
|
||||
: storage(reinterpret_cast<Storage const&>(value) & bits_mask_) {}
|
||||
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
integer_subbyte(unsigned value)
|
||||
: storage(reinterpret_cast<Storage const&>(value) & bits_mask_) {}
|
||||
|
||||
/// Convert to int or unsigned
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
operator xint_t() const {
|
||||
if (sign_mask_ & storage) { // Sign extend
|
||||
return xint_t(storage) | ~xint_t(bits_mask_);
|
||||
} else {
|
||||
return xint_t(storage);
|
||||
}
|
||||
}
|
||||
|
||||
/// Equality
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
bool operator==(integer_subbyte const& rhs) const {
|
||||
return storage == rhs.storage;
|
||||
}
|
||||
|
||||
/// Inequality
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
bool operator!=(integer_subbyte const& rhs) const {
|
||||
return storage != rhs.storage;
|
||||
}
|
||||
|
||||
/// Less than or equal
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
bool operator<=(integer_subbyte const& rhs) const {
|
||||
if (sign_mask_ & storage) {
|
||||
return !(rhs.storage < storage);
|
||||
} else {
|
||||
return storage <= rhs.storage;
|
||||
}
|
||||
}
|
||||
|
||||
/// Less than
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
bool operator<(integer_subbyte const& rhs) const {
|
||||
if (sign_mask_ & storage) {
|
||||
return !(rhs.storage <= storage);
|
||||
} else {
|
||||
return storage < rhs.storage;
|
||||
}
|
||||
}
|
||||
|
||||
/// Greater than or equal
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
bool operator>=(integer_subbyte const& rhs) const {
|
||||
return !(*this < rhs);
|
||||
}
|
||||
|
||||
/// Greater than
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
bool operator>(integer_subbyte const& rhs) const {
|
||||
return !(*this <= rhs);
|
||||
}
|
||||
};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// 1-bit unsigned integer type
|
||||
using uint1b_t = integer_subbyte<1, false>;
|
||||
|
||||
/// 2-bit integer type
|
||||
using int2b_t = integer_subbyte<2, true>;
|
||||
|
||||
/// 2-bit unsigned integer type
|
||||
using uint2b_t = integer_subbyte<2, false>;
|
||||
|
||||
/// 4-bit integer type
|
||||
using int4b_t = integer_subbyte<4, true>;
|
||||
|
||||
/// 4-bit unsigned integer type
|
||||
using uint4b_t = integer_subbyte<4, false>;
|
||||
|
||||
/// 1-bit binary type
|
||||
using bin1_t = bool;
|
||||
|
||||
} // namespace cute
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if !defined(__CUDACC_RTC__)
|
||||
|
||||
#include <limits>
|
||||
|
||||
namespace CUTE_STL_NAMESPACE {
|
||||
|
||||
template <>
|
||||
struct numeric_limits<cute::uint1b_t> {
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::uint1b_t const lowest() noexcept { return 0; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::uint1b_t const min() noexcept { return 0; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::uint1b_t const max() noexcept { return 1; }
|
||||
static constexpr bool is_integer = true;
|
||||
static constexpr bool is_signed = false;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct numeric_limits<cute::int2b_t> {
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::int2b_t lowest() noexcept { return -2; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::int2b_t min() noexcept { return -2; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::int2b_t max() noexcept { return 1; }
|
||||
static constexpr bool is_integer = true;
|
||||
static constexpr bool is_signed = true;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct numeric_limits<cute::uint2b_t> {
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::uint2b_t const lowest() noexcept { return 0; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::uint2b_t const min() noexcept { return 0; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::uint2b_t const max() noexcept { return 3; }
|
||||
static constexpr bool is_integer = true;
|
||||
static constexpr bool is_signed = false;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct numeric_limits<cute::int4b_t> {
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::int4b_t lowest() noexcept { return -8; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::int4b_t min() noexcept { return -8; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::int4b_t max() noexcept { return 7; }
|
||||
static constexpr bool is_integer = true;
|
||||
static constexpr bool is_signed = true;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct numeric_limits<cute::uint4b_t> {
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::uint4b_t const lowest() noexcept { return 0; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::uint4b_t const min() noexcept { return 0; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::uint4b_t const max() noexcept { return 15; }
|
||||
static constexpr bool is_integer = true;
|
||||
static constexpr bool is_signed = false;
|
||||
};
|
||||
|
||||
} // namespace std
|
||||
|
||||
#endif // !defined(__CUDACC_RTC__)
|
||||
@@ -443,4 +443,35 @@ CUTE_HOST std::ostream& operator<<(std::ostream& os, C<t> const&) {
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
namespace detail {
|
||||
|
||||
// parse_int_digits takes a variadic number of digits and converts them into an int
|
||||
template <class... Ts>
|
||||
constexpr uint64_t parse_int_digits(uint64_t result, int digit, Ts... digits)
|
||||
{
|
||||
if constexpr (sizeof...(Ts) == 0) {
|
||||
return 10 * result + digit;
|
||||
} else {
|
||||
return parse_int_digits(10 * result + digit, digits...);
|
||||
}
|
||||
}
|
||||
|
||||
} // end namespace detail
|
||||
|
||||
|
||||
// This user-defined literal operator allows cute::constant written as literals. For example,
|
||||
//
|
||||
// auto var = 32_c;
|
||||
//
|
||||
// var has type cute::constant<int,32>.
|
||||
//
|
||||
template <char... digits>
|
||||
constexpr cute::constant<int,detail::parse_int_digits(0, (digits - '0')...)> operator "" _c()
|
||||
{
|
||||
static_assert((('0' <= digits && digits <= '9') && ...),
|
||||
"Expected 0 <= digit <= 9 for each digit of the integer.");
|
||||
return {};
|
||||
}
|
||||
|
||||
} // end namespace cute
|
||||
|
||||
@@ -130,6 +130,10 @@ nratio(R<a,b>, R<c,d>) {
|
||||
return {};
|
||||
}
|
||||
|
||||
//
|
||||
// Operators
|
||||
//
|
||||
|
||||
template <auto a, auto b, auto x, auto y>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
typename R<a*x,b*y>::type
|
||||
@@ -227,14 +231,14 @@ abs(R<a,b>) {
|
||||
|
||||
template <auto a, auto b>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
int32_t
|
||||
log_2(R<a,b>) {
|
||||
static_assert(R<a,b>::num > 0);
|
||||
static_assert(R<a,b>::den > 0);
|
||||
return log_2(static_cast<uint32_t>(R<a,b>::num)) - log_2(static_cast<uint32_t>(R<a,b>::den));
|
||||
}
|
||||
|
||||
|
||||
// @return A non-reduced ratio cute::R of the Trait0::value / Trait1::value
|
||||
template <class Trait0, class Trait1>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
|
||||
@@ -316,11 +316,11 @@ safe_div(T const& t, U const& u) {
|
||||
|
||||
template <class T>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
int32_t
|
||||
log_2(T x) {
|
||||
assert(x > 0);
|
||||
static_assert(is_unsigned<T>::value, "Only to be used for unsigned integral types.");
|
||||
return bit_width(x) - 1;
|
||||
return static_cast<int32_t>(bit_width(x)) - 1;
|
||||
}
|
||||
|
||||
} // namespace cute
|
||||
|
||||
@@ -30,14 +30,46 @@
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#include <cute/config.hpp>
|
||||
|
||||
#include <vector_types.h>
|
||||
#include <cutlass/numeric_types.h>
|
||||
#include <cutlass/numeric_size.h>
|
||||
|
||||
#include <cute/numeric/int.hpp>
|
||||
#include <cute/numeric/real.hpp>
|
||||
|
||||
namespace cute {
|
||||
|
||||
template <typename T>
|
||||
struct sizeof_bits : public cutlass::sizeof_bits<T> {};
|
||||
|
||||
// DO NOT change auto to int, sizeof_bits<sparse_elem> use integral_ratio instead of int
|
||||
template <class T>
|
||||
static constexpr auto sizeof_bits_v = sizeof_bits<T>::value;
|
||||
|
||||
using cutlass::bits_to_bytes;
|
||||
|
||||
using cutlass::is_subbyte;
|
||||
|
||||
template <class T>
|
||||
static constexpr auto is_subbyte_v = is_subbyte<T>::value;
|
||||
|
||||
using cutlass::half_t;
|
||||
using cutlass::bfloat16_t;
|
||||
|
||||
using cutlass::tfloat32_t;
|
||||
|
||||
// Umbrella floating-point 8-bit data type : type_erased_dynamic_float8_t
|
||||
// This umbrella datatype can be enabled when a user provides a specific
|
||||
// datatype in runtime argument list.
|
||||
using cutlass::type_erased_dynamic_float8_t;
|
||||
using cutlass::float_e4m3_t;
|
||||
using cutlass::float_e5m2_t;
|
||||
|
||||
using cutlass::uint1b_t;
|
||||
using cutlass::int2b_t;
|
||||
using cutlass::uint2b_t;
|
||||
using cutlass::int4b_t;
|
||||
using cutlass::uint4b_t;
|
||||
using cutlass::bin1_t;
|
||||
|
||||
} // end namespace cute
|
||||
@@ -1,53 +0,0 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#include <cute/config.hpp>
|
||||
|
||||
#include <vector_types.h>
|
||||
#include <cutlass/numeric_types.h>
|
||||
|
||||
namespace cute {
|
||||
|
||||
using cutlass::tfloat32_t;
|
||||
|
||||
//
|
||||
// Display utilities
|
||||
//
|
||||
|
||||
#if !defined(__CUDACC_RTC__)
|
||||
CUTE_HOST std::ostream& operator<<(std::ostream& os, tfloat32_t const& v)
|
||||
{
|
||||
return os << float(v);
|
||||
}
|
||||
#endif
|
||||
|
||||
} // end namespace cute
|
||||
@@ -1,259 +0,0 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#if defined(__CUDACC_RTC__)
|
||||
#include <cuda/std/cstdint>
|
||||
#else
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <cmath>
|
||||
#include <type_traits>
|
||||
#include <stdexcept>
|
||||
#endif
|
||||
|
||||
#include <cute/config.hpp>
|
||||
|
||||
/// Optionally enable GCC's built-in type
|
||||
#if defined(__x86_64) && !defined(__CUDA_ARCH__)
|
||||
# if defined(__GNUC__) && 0
|
||||
# define CUTE_UINT128_NATIVE
|
||||
# elif defined(_MSC_VER)
|
||||
# define CUTE_INT128_ARITHMETIC
|
||||
# include <intrin.h>
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cute {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
///! Unsigned 128b integer type
|
||||
struct alignas(16) uint128_t
|
||||
{
|
||||
/// Size of one part of the uint's storage in bits
|
||||
static constexpr int storage_bits_ = 64;
|
||||
|
||||
struct hilo
|
||||
{
|
||||
uint64_t lo;
|
||||
uint64_t hi;
|
||||
};
|
||||
|
||||
// Use a union to store either low and high parts or, if present, a built-in 128b integer type.
|
||||
union
|
||||
{
|
||||
struct hilo hilo_;
|
||||
|
||||
#if defined(CUTE_UINT128_NATIVE)
|
||||
unsigned __int128 native;
|
||||
#endif // defined(CUTE_UINT128_NATIVE)
|
||||
};
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Default ctor
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint128_t() : hilo_{0, 0} {}
|
||||
|
||||
/// Constructor from uint64
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint128_t(uint64_t lo_) : hilo_{lo_, 0} {}
|
||||
|
||||
/// Constructor from two 64b unsigned integers
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint128_t(uint64_t lo_, uint64_t hi_) : hilo_{lo_, hi_} {}
|
||||
|
||||
/// Optional constructor from native value
|
||||
#if defined(CUTE_UINT128_NATIVE)
|
||||
uint128_t(unsigned __int128 value) : native(value) { }
|
||||
#endif
|
||||
|
||||
/// Lossily cast to uint64
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
explicit operator uint64_t() const
|
||||
{
|
||||
return hilo_.lo;
|
||||
}
|
||||
|
||||
template <class Dummy = bool>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
static void exception()
|
||||
{
|
||||
//static_assert(sizeof(Dummy) == 0, "Not implemented exception!");
|
||||
//abort();
|
||||
//printf("uint128 not implemented!\n");
|
||||
}
|
||||
|
||||
/// Add
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint128_t operator+(uint128_t const& rhs) const
|
||||
{
|
||||
uint128_t y;
|
||||
#if defined(CUTE_UINT128_NATIVE)
|
||||
y.native = native + rhs.native;
|
||||
#else
|
||||
y.hilo_.lo = hilo_.lo + rhs.hilo_.lo;
|
||||
y.hilo_.hi = hilo_.hi + rhs.hilo_.hi + (!y.hilo_.lo && (rhs.hilo_.lo));
|
||||
#endif
|
||||
return y;
|
||||
}
|
||||
|
||||
/// Subtract
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint128_t operator-(uint128_t const& rhs) const
|
||||
{
|
||||
uint128_t y;
|
||||
#if defined(CUTE_UINT128_NATIVE)
|
||||
y.native = native - rhs.native;
|
||||
#else
|
||||
y.hilo_.lo = hilo_.lo - rhs.hilo_.lo;
|
||||
y.hilo_.hi = hilo_.hi - rhs.hilo_.hi - (rhs.hilo_.lo && y.hilo_.lo > hilo_.lo);
|
||||
#endif
|
||||
return y;
|
||||
}
|
||||
|
||||
/// Multiply by unsigned 64b integer yielding 128b integer
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint128_t operator*(uint64_t const& rhs) const
|
||||
{
|
||||
uint128_t y;
|
||||
#if defined(CUTE_UINT128_NATIVE)
|
||||
y.native = native * rhs;
|
||||
#elif defined(CUTE_INT128_ARITHMETIC)
|
||||
// Multiply by the low part
|
||||
y.hilo_.lo = _umul128(hilo_.lo, rhs, &y.hilo_.hi);
|
||||
|
||||
// Add the high part and ignore the overflow
|
||||
uint64_t overflow;
|
||||
y.hilo_.hi += _umul128(hilo_.hi, rhs, &overflow);
|
||||
#else
|
||||
exception();
|
||||
#endif
|
||||
return y;
|
||||
}
|
||||
|
||||
/// Divide 128b operation by 64b operation yielding a 64b quotient
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint64_t operator/(uint64_t const& divisor) const
|
||||
{
|
||||
uint64_t quotient = 0;
|
||||
#if defined(CUTE_UINT128_NATIVE)
|
||||
quotient = uint64_t(native / divisor);
|
||||
#elif defined(CUTE_INT128_ARITHMETIC)
|
||||
// implemented using MSVC's arithmetic intrinsics
|
||||
uint64_t remainder = 0;
|
||||
quotient = _udiv128(hilo_.hi, hilo_.lo, divisor, &remainder);
|
||||
#else
|
||||
exception();
|
||||
#endif
|
||||
return quotient;
|
||||
}
|
||||
|
||||
/// Divide 128b operation by 64b operation yielding a 64b quotient
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint64_t operator%(uint64_t const& divisor) const
|
||||
{
|
||||
uint64_t remainder = 0;
|
||||
#if defined(CUTE_UINT128_NATIVE)
|
||||
remainder = uint64_t(native % divisor);
|
||||
#elif defined(CUTE_INT128_ARITHMETIC)
|
||||
// implemented using MSVC's arithmetic intrinsics
|
||||
(void)_udiv128(hilo_.hi, hilo_.lo, divisor, &remainder);
|
||||
#else
|
||||
exception();
|
||||
#endif
|
||||
return remainder;
|
||||
}
|
||||
|
||||
/// Computes the quotient and remainder in a single method.
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint64_t divmod(uint64_t &remainder, uint64_t divisor) const
|
||||
{
|
||||
uint64_t quotient = 0;
|
||||
#if defined(CUTE_UINT128_NATIVE)
|
||||
quotient = uint64_t(native / divisor);
|
||||
remainder = uint64_t(native % divisor);
|
||||
#elif defined(CUTE_INT128_ARITHMETIC)
|
||||
// implemented using MSVC's arithmetic intrinsics
|
||||
quotient = _udiv128(hilo_.hi, hilo_.lo, divisor, &remainder);
|
||||
#else
|
||||
exception();
|
||||
#endif
|
||||
return quotient;
|
||||
}
|
||||
|
||||
/// Left-shifts a 128b unsigned integer
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint128_t operator<<(int sh) const
|
||||
{
|
||||
if (sh == 0) {
|
||||
return *this;
|
||||
}
|
||||
else if (sh >= storage_bits_) {
|
||||
return uint128_t(0, hilo_.lo << (sh - storage_bits_));
|
||||
}
|
||||
else {
|
||||
return uint128_t(
|
||||
(hilo_.lo << sh),
|
||||
(hilo_.hi << sh) | uint64_t(hilo_.lo >> (storage_bits_ - sh))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Right-shifts a 128b unsigned integer
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint128_t operator>>(int sh) const
|
||||
{
|
||||
if (sh == 0) {
|
||||
return *this;
|
||||
}
|
||||
else if (sh >= storage_bits_) {
|
||||
return uint128_t((hilo_.hi >> (sh - storage_bits_)), 0);
|
||||
}
|
||||
else {
|
||||
return uint128_t(
|
||||
(hilo_.lo >> sh) | (hilo_.hi << (storage_bits_ - sh)),
|
||||
(hilo_.hi >> sh)
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace cute
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
Reference in New Issue
Block a user