CUTLASS 3.5.1 (#1623)

* CUTLASS 3.5.1

* updates, optimizations, fixes

include/cute/container/tuple.hpp

@@ -36,10 +36,13 @@
 #include <cute/numeric/integer_sequence.hpp>
 
 #include <cute/container/cuda_types.hpp>
+#include <cute/container/type_list.hpp>
+#if defined(CUTLASS_USE_PACKED_TUPLE)
+#  include <cute/container/packed_tuple.hpp>
+#endif
 
 //#include <cute/container/array.hpp>            // Advanced optimizations
 
-//
 // cute::tuple is like std::tuple, with two differences.
 //
 // 1. It works on both host and device.
@@ -50,19 +53,30 @@
 // but do _not_ include references like int& or float&.
 // (See std::tie for an example of a tuple of references.)
 //
-// This is simplified over the implementations in std::, cuda::std::, and thrust:: by ignoring much of
-//    the conversion SFINAE, special overloading, and avoiding cvref template types.
-//    Furthermore, the empty base optimization (EBO) is MORE aggressive by avoiding
-//    construction calls, and ignoring any need for unique element addresses.
-//
-// Over standard-conforming tuple implementations, this appears to accelerate compilation times by over 3x.
+// If the template arguments of cute::tuple are all empty types (in
+// the sense of std::is_empty_v), then the cute::tuple is also an
+// empty type.  Furthermore, if CUTLASS_USE_PACKED_TUPLE is defined,
+// cute::tuple is always a standard-layout type if all of its template
+// arguments are standard-layout types.
 
 namespace cute
 {
 
+#if defined(CUTLASS_USE_PACKED_TUPLE)
+
+template<class... T>
+using tuple = packed_tuple<T...>;
+
+#else
+
 namespace detail
 {
 
+// This is simplified over the implementations in std::, cuda::std::, and thrust:: by ignoring much of
+// the conversion SFINAE, special overloading, and avoiding cvref template types.
+//
+// Over standard-conforming tuple implementations, this appears to accelerate compilation times by over 3x.
+
 // EBO stands for "empty base optimization."
 // We use this technique to ensure that cute::tuple
 // doesn't need to waste space storing any template arguments
@@ -70,6 +84,12 @@ namespace detail
 // Otherwise, cute::tuple would need to spend at least 1 byte
 // for each of its template arguments.
 //
+// This is one way in which cute::tuple differs from std::tuple.
+// Empty types in the template argument list are not even constructed,
+// and do not have unique element addresses.  In fact, they are not
+// even members of the tuple or stored in any way.  Calling `get`
+// constructs and returns an instance of an empty type on demand.
+//
 // EBO always "holds" a single value of type T.
 // N is like an array index that TupleBase uses
 // to access the desired tuple element.
@@ -109,9 +129,8 @@ struct EBO<N, T, false>
   CUTE_HOST_DEVICE constexpr
   EBO() : t_{} {}
 
-  template <class U>
   CUTE_HOST_DEVICE constexpr
-  EBO(U const& u) : t_{u} {}
+  EBO(T const& t) : t_{t} {}
 
   T t_;
 };
@@ -141,15 +160,8 @@ struct TupleBase<index_sequence<I...>, T...>
   CUTE_HOST_DEVICE constexpr
   TupleBase() {}
 
-  template <class... U>
-  CUTE_HOST_DEVICE constexpr explicit
-  TupleBase(U const&... u)
-      : EBO<I,T>(u)... {}
-
-  template <class... U>
   CUTE_HOST_DEVICE constexpr
-  TupleBase(TupleBase<index_sequence<I...>, U...> const& u)
-      : EBO<I,T>(getv(static_cast<EBO<I,U> const&>(u)))... {}
+  TupleBase(T const&... t) : EBO<I,T>(t)... {}
 };
 
 } // end namespace detail
@@ -172,16 +184,14 @@ struct tuple : detail::TupleBase<make_index_sequence<sizeof...(T)>, T...>
   CUTE_HOST_DEVICE constexpr
   tuple() {}
 
-  template <class... U>
   CUTE_HOST_DEVICE constexpr
-  tuple(U const&... u) : detail::TupleBase<make_index_sequence<sizeof...(T)>, T...>(u...) {}
-
-  template <class... U>
-  CUTE_HOST_DEVICE constexpr
-  tuple(tuple<U...> const& u)
-      : detail::TupleBase<make_index_sequence<sizeof...(T)>, T...>(static_cast<detail::TupleBase<make_index_sequence<sizeof...(U)>, U...> const&>(u)) {}
+  tuple(T const&... t) : detail::TupleBase<make_index_sequence<sizeof...(T)>, T...>(t...) {}
 };
 
+template <>
+struct tuple<>
+{};
+
 //
 // get for cute::tuple (just like std::get for std::tuple)
 //
@@ -227,6 +237,8 @@ find(tuple<T...> const& t) noexcept
   return detail::findt<X>(t);
 }
 
+#endif // CUTLASS_USE_PACKED_TUPLE
+
 //
 // Custom is_tuple trait simply checks the existence of tuple_size
 //      and assumes std::get<I>(.), std::tuple_element<I,.>
@@ -242,6 +254,9 @@ auto has_tuple_size(...) -> false_type;
 template <class T>
 struct is_tuple : decltype(detail::has_tuple_size((T*)0)) {};
 
+template<typename T>
+constexpr bool is_tuple_v = cute::is_tuple<T>::value;
+
 //
 // make_tuple (value-based implementation)
 //
@@ -540,20 +555,12 @@ tuple_cat(Tuples const&... ts)
 
 namespace detail {
 
-template <size_t I, class TupleA, class TupleB>
+template <class TupleA, class TupleB, size_t... I>
 CUTE_HOST_DEVICE constexpr
 auto
-equal_impl(TupleA const& a, TupleB const& b)
+equal_impl(TupleA const& a, TupleB const& b, index_sequence<I...>)
 {
-  if constexpr (I == tuple_size<TupleA>::value) {
-    return cute::true_type{};   // Terminal: TupleA is exhausted
-  } else if constexpr (I == tuple_size<TupleB>::value) {
-    return cute::false_type{};  // Terminal: TupleA is not exhausted, TupleB is exhausted
-  } else {
-    return (get<I>(a) == get<I>(b)) && equal_impl<I+1>(a,b);
-  }
-
-  CUTE_GCC_UNREACHABLE;
+  return (cute::true_type{} && ... && (get<I>(a) == get<I>(b)));
 }
 
 } // end namespace detail
@@ -564,7 +571,13 @@ CUTE_HOST_DEVICE constexpr
 auto
 operator==(TupleT const& t, TupleU const& u)
 {
-  return detail::equal_impl<0>(t, u);
+  if constexpr (tuple_size<TupleT>::value == tuple_size<TupleU>::value) {
+    return detail::equal_impl(t, u, make_index_sequence<tuple_size<TupleT>::value>{});
+  } else {
+    return cute::false_type{};
+  }
+
+  CUTE_GCC_UNREACHABLE;
 }
 
 template <class TupleT, class TupleU,
@@ -618,19 +631,17 @@ operator!=(TupleT const& t, TupleU const& u)
 namespace detail {
 
 template <class Tuple, size_t... Is>
-CUTE_HOST_DEVICE void print_tuple(Tuple const& t,
-                                  index_sequence<Is...>, char s = '(', char e = ')')
+CUTE_HOST_DEVICE void print_tuple(Tuple const& t, index_sequence<Is...>, char s = '(', char e = ')')
 {
   using cute::print;
-  ((void(print(Is == 0 ? s : ',')), void(print(get<Is>(t)))), ...); print(e);
+  print(s); ((void(print(Is == 0 ? '\0' : ',')), void(print(get<Is>(t)))), ...); print(e);
 }
 
 #if !defined(__CUDACC_RTC__)
 template <class Tuple, std::size_t... Is>
-CUTE_HOST std::ostream& print_tuple_os(std::ostream& os, Tuple const& t,
-                                       index_sequence<Is...>, char s = '(', char e = ')')
+CUTE_HOST std::ostream& print_tuple_os(std::ostream& os, Tuple const& t, index_sequence<Is...>, char s = '(', char e = ')')
 {
-  (void(os << (Is == 0 ? s : ',') << get<Is>(t)), ...);
+  os << s; (void(os << (Is == 0 ? '\0' : ',') << get<Is>(t)), ...);
   return os << e;
 }
 #endif // !defined(__CUDACC_RTC__)
@@ -655,6 +666,8 @@ CUTE_HOST std::ostream& operator<<(std::ostream& os, Tuple const& t)
 
 } // end namespace cute
 
+#if ! defined(CUTLASS_USE_PACKED_TUPLE)
+
 namespace CUTE_STL_NAMESPACE
 {
 
@@ -716,5 +729,7 @@ struct tuple_element<I, const cute::tuple<T...>>
     : CUTE_STL_NAMESPACE::tuple_element<I, const CUTE_STL_NAMESPACE::tuple<T...>>
 {};
 
-} // end namepsace std
+} // end namespace std
 #endif // CUTE_STL_NAMESPACE_IS_CUDA_STD
+
+#endif // CUTLASS_USE_PACKED_TUPLE