leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 26 Packages Projects Releases Wiki Activity

CUTLASS 2.5

Browse Source
This commit is contained in:
Andrew Kerr
2021-02-26 09:58:26 -05:00
parent ccb697bac7
commit 0e13748649
771 changed files with 15474 additions and 1715 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/library/CMakeLists.txt
+1 -1
View File
@@ -1,4 +1,4 @@
# Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
# Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are permitted
# provided that the following conditions are met:
tools/library/include/cutlass/library/handle.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/library/include/cutlass/library/library.h
+6 -42
View File
@@ -571,7 +571,6 @@ struct ConvDescription : public OperationDescription {
};
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Base class for all operations
@@ -933,49 +932,14 @@ struct Conv2dConfiguration {
// also includes (split_k_slices, groups)
conv::Conv2dProblemSize problem_size;
/// Layout object for activations tensor
layout::TensorNHWC layout_activations;
// stride of operand A
std::vector<int> stride_a;
/// Layout object for filters tensor
layout::TensorNHWC layout_filters;
// stride of operand B
std::vector<int> stride_b;
/// Layout object for source tensor
layout::TensorNHWC layout_source;
/// Layout object for output tensor
layout::TensorNHWC layout_output;
//
// Methods
//
// Mapping functions (A,B,C -> activation,filter,output)
layout::TensorNHWC layout_a(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return layout_activations;
case library::ConvKind::kDgrad: return layout_output;
case library::ConvKind::kWgrad: return layout_output;
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
layout::TensorNHWC layout_b(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return layout_filters;
case library::ConvKind::kDgrad: return layout_filters;
case library::ConvKind::kWgrad: return layout_activations;
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
layout::TensorNHWC layout_c(library::ConvKind const &conv_kind) const {
switch (conv_kind) {
case library::ConvKind::kFprop: return layout_output;
case library::ConvKind::kDgrad: return layout_activations;
case library::ConvKind::kWgrad: return layout_filters;
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
}
}
// stride of operand C
std::vector<int> stride_c;
};
tools/library/include/cutlass/library/manifest.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/library/scripts/generator.py
+16 -17
View File
@@ -929,10 +929,10 @@ def GenerateSM75_TensorOp_8816_Interleaved(manifest, args):
operations = CreateGemmOperator(manifest, layouts, tile_descriptions, \
data_type_mixed, alignment_constraints, None, EpilogueFunctor.LinearCombinationClamp)
# conv_layout = (LayoutType.TensorNC32HW32, LayoutType.TensorC32RSK32, LayoutType.TensorNC32HW32)
#
# operations += CreateConv2dOperator(manifest, conv_layout, tile_descriptions,
# data_type_mixed, 16, [ConvKind.Fprop], EpilogueFunctor.LinearCombinationClamp)
conv_layout = (LayoutType.TensorNC32HW32, LayoutType.TensorC32RSK32, LayoutType.TensorNC32HW32)
operations += CreateConv2dOperator(manifest, conv_layout, tile_descriptions,
data_type_mixed, 16, [ConvKind.Fprop], EpilogueFunctor.LinearCombinationClamp)
for op in operations:
op.C.alignment = 8
@@ -1069,10 +1069,10 @@ def GenerateSM75_TensorOp_8832_Interleaved(manifest, args):
operations = CreateGemmOperator(manifest, layouts, tile_descriptions, \
data_type_mixed, alignment_constraints, None, EpilogueFunctor.LinearCombinationClamp)
# conv_layout = (LayoutType.TensorNC64HW64, LayoutType.TensorC64RSK64, LayoutType.TensorNC64HW64)
#
# operations += CreateConv2dOperator(manifest, conv_layout, tile_descriptions,
# data_type_mixed, 32, [ConvKind.Fprop], EpilogueFunctor.LinearCombinationClamp)
conv_layout = (LayoutType.TensorNC64HW64, LayoutType.TensorC64RSK64, LayoutType.TensorNC64HW64)
operations += CreateConv2dOperator(manifest, conv_layout, tile_descriptions,
data_type_mixed, 32, [ConvKind.Fprop], EpilogueFunctor.LinearCombinationClamp)
for op in operations:
op.C.alignment = 16
@@ -1644,10 +1644,10 @@ def GenerateSM80_TensorOp_16832_Interleaved(manifest, args):
operations = CreateGemmOperator(manifest, layouts, tile_descriptions, \
data_type_mixed, alignment_constraints, None, EpilogueFunctor.LinearCombinationClamp)
# conv_layout = (LayoutType.TensorNC32HW32, LayoutType.TensorC32RSK32, LayoutType.TensorNC32HW32)
#
# operations += CreateConv2dOperator(manifest, conv_layout, tile_descriptions,
# data_type_mixed, 16, [ConvKind.Fprop], EpilogueFunctor.LinearCombinationClamp)
conv_layout = (LayoutType.TensorNC32HW32, LayoutType.TensorC32RSK32, LayoutType.TensorNC32HW32)
operations += CreateConv2dOperator(manifest, conv_layout, tile_descriptions,
data_type_mixed, 16, [ConvKind.Fprop], EpilogueFunctor.LinearCombinationClamp)
for op in operations:
op.C.alignment = 8
@@ -1825,10 +1825,10 @@ def GenerateSM80_TensorOp_16864_Interleaved(manifest, args):
operations += CreateGemmOperator(manifest, layouts, tile_descriptions, \
data_type_mixed, alignment_constraints, None, EpilogueFunctor.LinearCombinationClamp)
# conv_layout = (LayoutType.TensorNC64HW64, LayoutType.TensorC64RSK64, LayoutType.TensorNC64HW64)
#
# operations += CreateConv2dOperator(manifest, conv_layout, tile_descriptions,
# data_type_mixed, 32, [ConvKind.Fprop], EpilogueFunctor.LinearCombinationClamp)
conv_layout = (LayoutType.TensorNC64HW64, LayoutType.TensorC64RSK64, LayoutType.TensorNC64HW64)
operations += CreateConv2dOperator(manifest, conv_layout, tile_descriptions,
data_type_mixed, 32, [ConvKind.Fprop], EpilogueFunctor.LinearCombinationClamp)
for op in operations:
op.C.alignment = 16
@@ -2096,7 +2096,6 @@ def GenerateSM80_TensorOp_1688_complex(manifest, args):
max_cc = 1024
tile_descriptions = [
TileDescription([64, 64, 16], 4, [2, 2, 1], math_inst, min_cc, max_cc),
TileDescription([128, 64, 16], 4, [4, 2, 1], math_inst, min_cc, max_cc),
TileDescription([64, 128, 16], 4, [2, 4, 1], math_inst, min_cc, max_cc),
TileDescription([64, 64, 16], 4, [2, 2, 1], math_inst, min_cc, max_cc),
tools/library/scripts/library.py
+3 -2
View File
@@ -187,7 +187,6 @@ DataTypeSize = {
}
###################################################################################################
#
class ComplexTransform(enum.Enum):
none = enum_auto()
@@ -312,7 +311,7 @@ TransposedLayout = {
#
ShortLayoutTypeNames = {
LayoutType.ColumnMajor: 'n',
LayoutType.ColumnMajorInterleaved32: 'n2',
LayoutType.ColumnMajorInterleaved2: 'n2',
LayoutType.ColumnMajorInterleaved32: 'n32',
LayoutType.ColumnMajorInterleaved64: 'n64',
LayoutType.RowMajor: 't',
@@ -343,6 +342,8 @@ class OpcodeClass(enum.Enum):
Simt = enum_auto()
TensorOp = enum_auto()
WmmaTensorOp = enum_auto()
SparseTensorOp = enum_auto()
OpcodeClassNames = {
OpcodeClass.Simt: 'simt',
tools/library/src/conv2d_operation.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/library/src/conv3d_operation.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/library/src/gemm_operation.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/library/src/handle.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/library/src/library_internal.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/library/src/manifest.cpp
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/library/src/reduction/init_reduction_operations.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/library/src/reduction/reduction_device.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/library/src/reference/conv2d.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/library/src/reference/conv3d.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/library/src/reference/conv_reference_operation.h
+22 -10
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
@@ -109,7 +109,19 @@ struct ConvReferenceDispatcher<
Conv2dConfiguration const &config =
*static_cast<Conv2dConfiguration const *>(configuration);
ConvKind const conv_kind = ConvKindMap<kConvolutionalOperator>::kId;
// TODO: make below code more general. It is fixed for NHWC now.
layout::TensorNHWC layout_a;
layout::TensorNHWC layout_b;
layout::TensorNHWC layout_c;
layout_a.stride() =
make_Coord(config.stride_a[0], config.stride_a[1], config.stride_a[2]);
layout_b.stride() =
make_Coord(config.stride_b[0], config.stride_b[1], config.stride_b[2]);
layout_c.stride() =
make_Coord(config.stride_c[0], config.stride_c[1], config.stride_c[2]);
if (kProvider == Provider::kReferenceHost) {
@@ -127,10 +139,10 @@ struct ConvReferenceDispatcher<
>(
kConvolutionalOperator,
config.problem_size,
{ptr_A, config.layout_a(conv_kind)},
{ptr_B, config.layout_b(conv_kind)},
{ptr_C, config.layout_c(conv_kind)},
{ptr_D, config.layout_c(conv_kind)},
{ptr_A, layout_a},
{ptr_B, layout_b},
{ptr_C, layout_c},
{ptr_D, layout_c},
alpha,
beta
);
@@ -152,10 +164,10 @@ struct ConvReferenceDispatcher<
>(
kConvolutionalOperator,
config.problem_size,
{ptr_A, config.layout_a(conv_kind)},
{ptr_B, config.layout_b(conv_kind)},
{ptr_C, config.layout_c(conv_kind)},
{ptr_D, config.layout_c(conv_kind)},
{ptr_A, layout_a},
{ptr_B, layout_b},
{ptr_C, layout_c},
{ptr_D, layout_c},
alpha,
beta,
stream
tools/library/src/reference/gemm.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/library/src/reference/gemm_reference_operation.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/library/src/reference/initialize_reference_operations.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met: