leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 27 Packages Projects Releases Wiki Activity

CUTLASS 3.8 Release (#2059)

Browse Source 
* CUTLASS 3.8 Release

* update

* Update README.md

* Revert "Update README.md"

This reverts commit b353e36fe83e0815f99b44e46c0c95494c44726b.

* update

* update

---------

Co-authored-by: Haicheng Wu <57973641+hwu36@users.noreply.github.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
mihir-awatramani
2025-01-25 02:44:06 -05:00
committed by GitHub
co-authored by Haicheng Wu Haicheng Wu
parent 9eb01fa0b0
commit 389e493055
290 changed files with 91222 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/profiler/src/device_allocation.cu
+325
View File
@@ -616,6 +616,48 @@ void DeviceAllocation::initialize_random_device(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kFUE4M3:
cutlass::reference::device::BlockFillRandom<cutlass::float_ue4m3_t>(
reinterpret_cast<cutlass::float_ue4m3_t *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kFUE8M0:
cutlass::reference::device::BlockFillRandom<cutlass::float_ue8m0_t>(
reinterpret_cast<cutlass::float_ue8m0_t *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kFE2M3:
cutlass::reference::device::BlockFillRandom<cutlass::float_e2m3_t>(
reinterpret_cast<cutlass::float_e2m3_t *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kFE3M2:
cutlass::reference::device::BlockFillRandom<cutlass::float_e3m2_t>(
reinterpret_cast<cutlass::float_e3m2_t *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kFE2M1:
cutlass::reference::device::BlockFillRandom<cutlass::float_e2m1_t>(
reinterpret_cast<cutlass::float_e2m1_t *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kF64:
cutlass::reference::device::BlockFillRandom<double>(
reinterpret_cast<double *>(pointer_),
@@ -771,6 +813,50 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kFUE4M3:
cutlass::reference::host::BlockFillRandom<cutlass::float_ue4m3_t>(
reinterpret_cast<cutlass::float_ue4m3_t *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kFE2M3:
cutlass::reference::host::BlockFillRandom<cutlass::float_e2m3_t>(
reinterpret_cast<cutlass::float_e2m3_t *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kFE3M2:
cutlass::reference::host::BlockFillRandom<cutlass::float_e3m2_t>(
reinterpret_cast<cutlass::float_e3m2_t *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kFE2M1:
cutlass::reference::host::BlockFillRandom<cutlass::float_e2m1_t>(
reinterpret_cast<cutlass::float_e2m1_t *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kFUE8M0:
cutlass::reference::host::BlockFillRandom<cutlass::float_ue8m0_t>(
reinterpret_cast<cutlass::float_ue8m0_t *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kF16:
cutlass::reference::host::BlockFillRandom<cutlass::half_t>(
reinterpret_cast<cutlass::half_t *>(host_data.data()),
@@ -990,6 +1076,50 @@ void DeviceAllocation::initialize_sequential_device(Distribution dist) {
static_cast<cutlass::float_e5m2_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFUE4M3:
cutlass::reference::device::BlockFillSequential<cutlass::float_ue4m3_t>(
reinterpret_cast<cutlass::float_ue4m3_t *>(pointer_),
capacity_,
static_cast<cutlass::float_ue4m3_t>(dist.sequential.delta),
static_cast<cutlass::float_ue4m3_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFE2M3:
cutlass::reference::device::BlockFillSequential<cutlass::float_e2m3_t>(
reinterpret_cast<cutlass::float_e2m3_t *>(pointer_),
capacity_,
static_cast<cutlass::float_e2m3_t>(dist.sequential.delta),
static_cast<cutlass::float_e2m3_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFE3M2:
cutlass::reference::device::BlockFillSequential<cutlass::float_e3m2_t>(
reinterpret_cast<cutlass::float_e3m2_t *>(pointer_),
capacity_,
static_cast<cutlass::float_e3m2_t>(dist.sequential.delta),
static_cast<cutlass::float_e3m2_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFE2M1:
cutlass::reference::device::BlockFillSequential<cutlass::float_e2m1_t>(
reinterpret_cast<cutlass::float_e2m1_t *>(pointer_),
capacity_,
static_cast<cutlass::float_e2m1_t>(dist.sequential.delta),
static_cast<cutlass::float_e2m1_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFUE8M0:
cutlass::reference::device::BlockFillSequential<cutlass::float_ue8m0_t>(
reinterpret_cast<cutlass::float_ue8m0_t *>(pointer_),
capacity_,
static_cast<cutlass::float_ue8m0_t>(dist.sequential.delta),
static_cast<cutlass::float_ue8m0_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kF16:
cutlass::reference::device::BlockFillSequential<cutlass::half_t>(
reinterpret_cast<cutlass::half_t *>(pointer_),
@@ -1220,6 +1350,50 @@ void DeviceAllocation::initialize_sequential_host(Distribution dist) {
static_cast<cutlass::float_e5m2_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFUE4M3:
cutlass::reference::host::BlockFillSequential<cutlass::float_ue4m3_t>(
reinterpret_cast<cutlass::float_ue4m3_t *>(host_data.data()),
capacity_,
static_cast<cutlass::float_ue4m3_t>(dist.sequential.delta),
static_cast<cutlass::float_ue4m3_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFE2M3:
cutlass::reference::host::BlockFillSequential<cutlass::float_e2m3_t>(
reinterpret_cast<cutlass::float_e2m3_t *>(host_data.data()),
capacity_,
static_cast<cutlass::float_e2m3_t>(dist.sequential.delta),
static_cast<cutlass::float_e2m3_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFE3M2:
cutlass::reference::host::BlockFillSequential<cutlass::float_e3m2_t>(
reinterpret_cast<cutlass::float_e3m2_t *>(host_data.data()),
capacity_,
static_cast<cutlass::float_e3m2_t>(dist.sequential.delta),
static_cast<cutlass::float_e3m2_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFE2M1:
cutlass::reference::host::BlockFillSequential<cutlass::float_e2m1_t>(
reinterpret_cast<cutlass::float_e2m1_t *>(host_data.data()),
capacity_,
static_cast<cutlass::float_e2m1_t>(dist.sequential.delta),
static_cast<cutlass::float_e2m1_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFUE8M0:
cutlass::reference::host::BlockFillSequential<cutlass::float_ue8m0_t>(
reinterpret_cast<cutlass::float_ue8m0_t *>(host_data.data()),
capacity_,
static_cast<cutlass::float_ue8m0_t>(dist.sequential.delta),
static_cast<cutlass::float_ue8m0_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kF16:
cutlass::reference::host::BlockFillSequential<cutlass::half_t>(
reinterpret_cast<cutlass::half_t *>(host_data.data()),
@@ -1516,6 +1690,34 @@ bool DeviceAllocation::block_compare_equal(
reinterpret_cast<float_e5m2_t const *>(ptr_A),
reinterpret_cast<float_e5m2_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kFUE4M3:
return reference::device::BlockCompareEqual<float_ue4m3_t>(
reinterpret_cast<float_ue4m3_t const *>(ptr_A),
reinterpret_cast<float_ue4m3_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kFUE8M0:
return reference::device::BlockCompareEqual<float_ue8m0_t>(
reinterpret_cast<float_ue8m0_t const *>(ptr_A),
reinterpret_cast<float_ue8m0_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kFE2M3:
return reference::device::BlockCompareEqual<float_e2m3_t>(
reinterpret_cast<float_e2m3_t const *>(ptr_A),
reinterpret_cast<float_e2m3_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kFE3M2:
return reference::device::BlockCompareEqual<float_e3m2_t>(
reinterpret_cast<float_e3m2_t const *>(ptr_A),
reinterpret_cast<float_e3m2_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kFE2M1:
return reference::device::BlockCompareEqual<float_e2m1_t>(
reinterpret_cast<float_e2m1_t const *>(ptr_A),
reinterpret_cast<float_e2m1_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kF16:
return reference::device::BlockCompareEqual<half_t>(
reinterpret_cast<half_t const *>(ptr_A),
@@ -1684,6 +1886,46 @@ bool DeviceAllocation::block_compare_relatively_equal(
capacity,
static_cast<float_e5m2_t>(epsilon),
static_cast<float_e5m2_t>(nonzero_floor));
case library::NumericTypeID::kFUE4M3:
return reference::device::BlockCompareRelativelyEqual<float_ue4m3_t>(
reinterpret_cast<float_ue4m3_t const *>(ptr_A),
reinterpret_cast<float_ue4m3_t const *>(ptr_B),
capacity,
static_cast<float_ue4m3_t>(epsilon),
static_cast<float_ue4m3_t>(nonzero_floor));
case library::NumericTypeID::kFUE8M0:
return reference::device::BlockCompareRelativelyEqual<float_ue8m0_t>(
reinterpret_cast<float_ue8m0_t const *>(ptr_A),
reinterpret_cast<float_ue8m0_t const *>(ptr_B),
capacity,
static_cast<float_ue8m0_t>(epsilon),
static_cast<float_ue8m0_t>(nonzero_floor));
case library::NumericTypeID::kFE2M3:
return reference::device::BlockCompareRelativelyEqual<float_e2m3_t>(
reinterpret_cast<float_e2m3_t const *>(ptr_A),
reinterpret_cast<float_e2m3_t const *>(ptr_B),
capacity,
static_cast<float_e2m3_t>(epsilon),
static_cast<float_e2m3_t>(nonzero_floor));
case library::NumericTypeID::kFE3M2:
return reference::device::BlockCompareRelativelyEqual<float_e3m2_t>(
reinterpret_cast<float_e3m2_t const *>(ptr_A),
reinterpret_cast<float_e3m2_t const *>(ptr_B),
capacity,
static_cast<float_e3m2_t>(epsilon),
static_cast<float_e3m2_t>(nonzero_floor));
case library::NumericTypeID::kFE2M1:
return reference::device::BlockCompareRelativelyEqual<float_e2m1_t>(
reinterpret_cast<float_e2m1_t const *>(ptr_A),
reinterpret_cast<float_e2m1_t const *>(ptr_B),
capacity,
static_cast<float_e2m1_t>(epsilon),
static_cast<float_e2m1_t>(nonzero_floor));
case library::NumericTypeID::kF16:
return reference::device::BlockCompareRelativelyEqual<half_t>(
reinterpret_cast<half_t const *>(ptr_A),
@@ -2026,6 +2268,27 @@ void DeviceAllocation::write_tensor_csv(
case library::NumericTypeID::kFE5M2:
write_tensor_csv_static_type<float_e5m2_t>(out, *this);
break;
case library::NumericTypeID::kFUE4M3:
write_tensor_csv_static_type<float_ue4m3_t>(out, *this);
break;
case library::NumericTypeID::kFE2M3:
write_tensor_csv_static_type<float_e2m3_t>(out, *this);
break;
case library::NumericTypeID::kFE3M2:
write_tensor_csv_static_type<float_e3m2_t>(out, *this);
break;
case library::NumericTypeID::kFE2M1:
write_tensor_csv_static_type<float_e2m1_t>(out, *this);
break;
case library::NumericTypeID::kFUE8M0:
write_tensor_csv_static_type<float_ue8m0_t>(out, *this);
break;
case library::NumericTypeID::kF16:
write_tensor_csv_static_type<half_t>(out, *this);
break;
@@ -2193,6 +2456,27 @@ void DeviceAllocation::fill_device(double val = 0.0) {
case library::NumericTypeID::kFE5M2:
tensor_fill<float_e5m2_t>(*this, static_cast<float_e5m2_t>(val));
break;
case library::NumericTypeID::kFUE4M3:
tensor_fill<float_ue4m3_t>(*this, static_cast<float_ue4m3_t>(val));
break;
case library::NumericTypeID::kFUE8M0:
tensor_fill<float_ue8m0_t>(*this, static_cast<float_ue8m0_t>(val));
break;
case library::NumericTypeID::kFE2M3:
tensor_fill<float_e2m3_t>(*this, static_cast<float_e2m3_t>(val));
break;
case library::NumericTypeID::kFE3M2:
tensor_fill<float_e3m2_t>(*this, static_cast<float_e3m2_t>(val));
break;
case library::NumericTypeID::kFE2M1:
tensor_fill<float_e2m1_t>(*this, static_cast<float_e2m1_t>(val));
break;
case library::NumericTypeID::kF16:
tensor_fill<half_t>(*this, static_cast<half_t>(val));
break;
@@ -2288,6 +2572,47 @@ void DeviceAllocation::fill_host(double val = 0.0) {
std::vector<uint8_t> host_data(bytes());
switch (this->type()) {
case library::NumericTypeID::kFUE4M3:
cutlass::reference::host::BlockFill<float_ue4m3_t>(
reinterpret_cast<float_ue4m3_t *>(host_data.data()),
capacity_,
static_cast<float_ue4m3_t>(val)
);
break;
case library::NumericTypeID::kFUE8M0:
cutlass::reference::host::BlockFill<float_ue8m0_t>(
reinterpret_cast<float_ue8m0_t *>(host_data.data()),
capacity_,
static_cast<float_ue8m0_t>(val)
);
break;
case library::NumericTypeID::kFE2M3:
cutlass::reference::host::BlockFill<float_e2m3_t>(
reinterpret_cast<float_e2m3_t *>(host_data.data()),
capacity_,
static_cast<float_e2m3_t>(val)
);
break;
case library::NumericTypeID::kFE3M2:
cutlass::reference::host::BlockFill<float_e3m2_t>(
reinterpret_cast<float_e3m2_t *>(host_data.data()),
capacity_,
static_cast<float_e3m2_t>(val)
);
break;
case library::NumericTypeID::kFE2M1:
cutlass::reference::host::BlockFill<float_e2m1_t>(
reinterpret_cast<float_e2m1_t *>(host_data.data()),
capacity_,
static_cast<float_e2m1_t>(val)
);
break;
case library::NumericTypeID::kFE4M3:
cutlass::reference::host::BlockFill<float_e4m3_t>(
reinterpret_cast<float_e4m3_t *>(host_data.data()),