CUTLASS 2.9 (#468)
This commit is contained in:
Andrew Kerr
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
@@ -170,6 +176,54 @@ bool get_cublas_datatype(cublasDataType_t &data_type, library::NumericTypeID ele
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Maps a cutlass::SideMode to cuBLAS side mode
|
||||
bool get_cublas_side_mode(cublasSideMode_t& side, SideMode side_mode) {
|
||||
|
||||
switch (side_mode) {
|
||||
case SideMode::kLeft:
|
||||
side = CUBLAS_SIDE_LEFT;
|
||||
return true;
|
||||
case SideMode::kRight:
|
||||
side = CUBLAS_SIDE_RIGHT;
|
||||
return true;
|
||||
default: break;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Maps a cutlass::FillMode to cuBLAS fill mode
|
||||
bool get_cublas_fill_mode(cublasFillMode_t& uplo, FillMode fill_mode) {
|
||||
|
||||
switch (fill_mode) {
|
||||
case FillMode::kLower:
|
||||
uplo = CUBLAS_FILL_MODE_LOWER;
|
||||
return true;
|
||||
case FillMode::kUpper:
|
||||
uplo = CUBLAS_FILL_MODE_UPPER;
|
||||
return true;
|
||||
default: break;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Maps a cutlass::DiagType to cuBLAS diag type
|
||||
bool get_cublas_diag_type(cublasDiagType_t& diag, DiagType diag_type) {
|
||||
|
||||
switch (diag_type) {
|
||||
case DiagType::kNonUnit:
|
||||
diag = CUBLAS_DIAG_NON_UNIT;
|
||||
return true;
|
||||
case DiagType::kUnit:
|
||||
diag = CUBLAS_DIAG_UNIT;
|
||||
return true;
|
||||
default: break;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Gets the cublas algorithm given threadblock tile dimensions and math opcode class
|
||||
@@ -332,6 +386,773 @@ cublasStatus_t cublasGemmExDispatcher::operator()(cublasHandle_t handle) {
|
||||
} // namespace detail
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Returns a status if cuBLAS can satisfy a particular RankK description
|
||||
Status cublas_satisfies(library::RankKDescription const &desc) {
|
||||
auto const &math_instruction = desc.tile_description.math_instruction;
|
||||
|
||||
if (math_instruction.element_accumulator == library::NumericTypeID::kS32 &&
|
||||
math_instruction.opcode_class == library::OpcodeClassID::kTensorOp) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
// output type S4 and S8 not supported in cuBLAS
|
||||
if (desc.C.element == library::NumericTypeID::kS4 ||
|
||||
desc.C.element == library::NumericTypeID::kS8) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
// input type BF16 and TF32 not supported in cuBLAS
|
||||
if (desc.A.element == library::NumericTypeID::kBF16 ||
|
||||
desc.A.element == library::NumericTypeID::kTF32) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace detail {
|
||||
|
||||
cublasRankKDispatcher::cublasRankKDispatcher(
|
||||
library::RankKDescription const &op_desc,
|
||||
library::RankKConfiguration configuration_,
|
||||
library::RankKArguments arguments_
|
||||
):
|
||||
configuration(configuration_), arguments(arguments_), status(Status::kSuccess) {
|
||||
|
||||
blas_mode = op_desc.blas_mode;
|
||||
num_ranks = op_desc.num_ranks;
|
||||
|
||||
bool good = true;
|
||||
|
||||
good = (good && get_cublas_transpose_operation(trans_A, op_desc.A.layout, op_desc.transform_A));
|
||||
good = (good && get_cublas_fill_mode(uplo, op_desc.fill_mode));
|
||||
good = (good && get_cublas_datatype(data_type_A, op_desc.A.element));
|
||||
good = (good && get_cublas_datatype(data_type_C, op_desc.C.element));
|
||||
|
||||
good = (good && get_cublas_datatype(
|
||||
compute_data_type,
|
||||
op_desc.tile_description.math_instruction.element_accumulator));
|
||||
|
||||
// cuBLAS introduces a separate cublasComputeType enumerant to more precisely describe
|
||||
// internal numerical data types used in the computation.
|
||||
#if (__CUDACC_VER_MAJOR__ >= 11)
|
||||
library::OpcodeClassID const & opcode_class =
|
||||
op_desc.tile_description.math_instruction.opcode_class;
|
||||
|
||||
if (good &&
|
||||
op_desc.A.element == library::NumericTypeID::kF32 &&
|
||||
opcode_class == library::OpcodeClassID::kTensorOp) {
|
||||
|
||||
compute_type = CUBLAS_COMPUTE_32F_FAST_TF32;
|
||||
}
|
||||
else if (good) {
|
||||
bool const isPedantic = false;
|
||||
switch (compute_data_type) {
|
||||
case CUDA_R_32F:
|
||||
case CUDA_C_32F:
|
||||
compute_type = isPedantic ? CUBLAS_COMPUTE_32F_PEDANTIC : CUBLAS_COMPUTE_32F;
|
||||
break;
|
||||
case CUDA_R_64F:
|
||||
case CUDA_C_64F:
|
||||
compute_type = isPedantic ? CUBLAS_COMPUTE_64F_PEDANTIC : CUBLAS_COMPUTE_64F;
|
||||
break;
|
||||
case CUDA_R_16F:
|
||||
compute_type = isPedantic ? CUBLAS_COMPUTE_16F_PEDANTIC : CUBLAS_COMPUTE_16F;
|
||||
break;
|
||||
case CUDA_R_32I:
|
||||
compute_type = isPedantic ? CUBLAS_COMPUTE_32I_PEDANTIC : CUBLAS_COMPUTE_32I;
|
||||
break;
|
||||
default:
|
||||
good = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
#endif // __CUDACC_VER_MAJOR__ >= 11
|
||||
|
||||
if (!good) {
|
||||
status = Status::kErrorNotSupported;
|
||||
}
|
||||
}
|
||||
|
||||
/// Executes RankK using these arguments
|
||||
cublasStatus_t cublasRankKDispatcher::operator()(cublasHandle_t handle) {
|
||||
|
||||
// SYRK and HERK
|
||||
if (num_ranks == 1) {
|
||||
if (data_type_A == data_type_C && data_type_A == CUDA_R_64F) {
|
||||
return cublasDsyrk(
|
||||
handle,
|
||||
uplo,
|
||||
trans_A,
|
||||
configuration.problem_size.n(),
|
||||
configuration.problem_size.k(),
|
||||
static_cast<const double*>(arguments.alpha),
|
||||
static_cast<const double*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const double*>(arguments.beta),
|
||||
static_cast<double*>(arguments.D),
|
||||
int(configuration.ldc)
|
||||
);
|
||||
} else if (data_type_A == data_type_C && data_type_A == CUDA_R_32F) {
|
||||
|
||||
#if (__CUDACC_VER_MAJOR__ >= 11)
|
||||
if (cublasSetMathMode(handle, CUBLAS_TF32_TENSOR_OP_MATH) != CUBLAS_STATUS_SUCCESS)
|
||||
return CUBLAS_STATUS_NOT_SUPPORTED;
|
||||
#endif
|
||||
|
||||
return cublasSsyrk(
|
||||
handle,
|
||||
uplo,
|
||||
trans_A,
|
||||
configuration.problem_size.n(),
|
||||
configuration.problem_size.k(),
|
||||
static_cast<const float*>(arguments.alpha),
|
||||
static_cast<const float*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const float*>(arguments.beta),
|
||||
static_cast<float*>(arguments.D),
|
||||
int(configuration.ldc)
|
||||
);
|
||||
} else if (data_type_A == data_type_C && data_type_A == CUDA_C_64F) {
|
||||
|
||||
if (blas_mode == BlasMode::kHermitian) {
|
||||
return cublasZherk(
|
||||
handle,
|
||||
uplo,
|
||||
trans_A,
|
||||
configuration.problem_size.n(),
|
||||
configuration.problem_size.k(),
|
||||
static_cast<const double*>(arguments.alpha),
|
||||
static_cast<const cuDoubleComplex*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const double*>(arguments.beta),
|
||||
static_cast<cuDoubleComplex*>(arguments.D),
|
||||
int(configuration.ldc)
|
||||
);
|
||||
}
|
||||
else {
|
||||
return cublasZsyrk(
|
||||
handle,
|
||||
uplo,
|
||||
trans_A,
|
||||
configuration.problem_size.n(),
|
||||
configuration.problem_size.k(),
|
||||
static_cast<const cuDoubleComplex*>(arguments.alpha),
|
||||
static_cast<const cuDoubleComplex*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const cuDoubleComplex*>(arguments.beta),
|
||||
static_cast<cuDoubleComplex*>(arguments.D),
|
||||
int(configuration.ldc)
|
||||
);
|
||||
}
|
||||
|
||||
} else if (data_type_A == data_type_C && data_type_A == CUDA_C_32F) {
|
||||
|
||||
#if (__CUDACC_VER_MAJOR__ >= 11)
|
||||
if (cublasSetMathMode(handle, CUBLAS_TF32_TENSOR_OP_MATH) != CUBLAS_STATUS_SUCCESS)
|
||||
return CUBLAS_STATUS_NOT_SUPPORTED;
|
||||
#endif
|
||||
|
||||
if (blas_mode == BlasMode::kHermitian) {
|
||||
return cublasCherk(
|
||||
handle,
|
||||
uplo,
|
||||
trans_A,
|
||||
configuration.problem_size.n(),
|
||||
configuration.problem_size.k(),
|
||||
static_cast<const float*>(arguments.alpha),
|
||||
static_cast<const cuComplex*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const float*>(arguments.beta),
|
||||
static_cast<cuComplex*>(arguments.D),
|
||||
int(configuration.ldc)
|
||||
);
|
||||
}
|
||||
else {
|
||||
return cublasCsyrk(
|
||||
handle,
|
||||
uplo,
|
||||
trans_A,
|
||||
configuration.problem_size.n(),
|
||||
configuration.problem_size.k(),
|
||||
static_cast<const cuComplex*>(arguments.alpha),
|
||||
static_cast<const cuComplex*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const cuComplex*>(arguments.beta),
|
||||
static_cast<cuComplex*>(arguments.D),
|
||||
int(configuration.ldc)
|
||||
);
|
||||
}
|
||||
} else {
|
||||
return CUBLAS_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
}
|
||||
|
||||
// SYR2K and HER2K
|
||||
else if (num_ranks == 2) {
|
||||
if (data_type_A == data_type_C && data_type_A == CUDA_R_64F) {
|
||||
return cublasDsyr2k(
|
||||
handle,
|
||||
uplo,
|
||||
trans_A,
|
||||
configuration.problem_size.n(),
|
||||
configuration.problem_size.k(),
|
||||
static_cast<const double*>(arguments.alpha),
|
||||
static_cast<const double*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const double*>(arguments.B),
|
||||
int(configuration.ldb),
|
||||
static_cast<const double*>(arguments.beta),
|
||||
static_cast<double*>(arguments.D),
|
||||
int(configuration.ldc)
|
||||
);
|
||||
} else if (data_type_A == data_type_C && data_type_A == CUDA_R_32F) {
|
||||
|
||||
#if (__CUDACC_VER_MAJOR__ >= 11)
|
||||
if (cublasSetMathMode(handle, CUBLAS_TF32_TENSOR_OP_MATH) != CUBLAS_STATUS_SUCCESS)
|
||||
return CUBLAS_STATUS_NOT_SUPPORTED;
|
||||
#endif
|
||||
|
||||
return cublasSsyr2k(
|
||||
handle,
|
||||
uplo,
|
||||
trans_A,
|
||||
configuration.problem_size.n(),
|
||||
configuration.problem_size.k(),
|
||||
static_cast<const float*>(arguments.alpha),
|
||||
static_cast<const float*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const float*>(arguments.B),
|
||||
int(configuration.ldb),
|
||||
static_cast<const float*>(arguments.beta),
|
||||
static_cast<float*>(arguments.D),
|
||||
int(configuration.ldc)
|
||||
);
|
||||
} else if (data_type_A == data_type_C && data_type_A == CUDA_C_64F) {
|
||||
|
||||
if (blas_mode == BlasMode::kHermitian) {
|
||||
return cublasZher2k(
|
||||
handle,
|
||||
uplo,
|
||||
trans_A,
|
||||
configuration.problem_size.n(),
|
||||
configuration.problem_size.k(),
|
||||
static_cast<const cuDoubleComplex*>(arguments.alpha),
|
||||
static_cast<const cuDoubleComplex*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const cuDoubleComplex*>(arguments.B),
|
||||
int(configuration.ldb),
|
||||
static_cast<const double*>(arguments.beta),
|
||||
static_cast<cuDoubleComplex*>(arguments.D),
|
||||
int(configuration.ldc)
|
||||
);
|
||||
}
|
||||
else {
|
||||
return cublasZsyr2k(
|
||||
handle,
|
||||
uplo,
|
||||
trans_A,
|
||||
configuration.problem_size.n(),
|
||||
configuration.problem_size.k(),
|
||||
static_cast<const cuDoubleComplex*>(arguments.alpha),
|
||||
static_cast<const cuDoubleComplex*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const cuDoubleComplex*>(arguments.B),
|
||||
int(configuration.ldb),
|
||||
static_cast<const cuDoubleComplex*>(arguments.beta),
|
||||
static_cast<cuDoubleComplex*>(arguments.D),
|
||||
int(configuration.ldc)
|
||||
);
|
||||
}
|
||||
|
||||
} else if (data_type_A == data_type_C && data_type_A == CUDA_C_32F) {
|
||||
|
||||
#if (__CUDACC_VER_MAJOR__ >= 11)
|
||||
if (cublasSetMathMode(handle, CUBLAS_TF32_TENSOR_OP_MATH) != CUBLAS_STATUS_SUCCESS)
|
||||
return CUBLAS_STATUS_NOT_SUPPORTED;
|
||||
#endif
|
||||
|
||||
if (blas_mode == BlasMode::kHermitian) {
|
||||
return cublasCher2k(
|
||||
handle,
|
||||
uplo,
|
||||
trans_A,
|
||||
configuration.problem_size.n(),
|
||||
configuration.problem_size.k(),
|
||||
static_cast<const cuComplex*>(arguments.alpha),
|
||||
static_cast<const cuComplex*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const cuComplex*>(arguments.B),
|
||||
int(configuration.ldb),
|
||||
static_cast<const float*>(arguments.beta),
|
||||
static_cast<cuComplex*>(arguments.D),
|
||||
int(configuration.ldc)
|
||||
);
|
||||
}
|
||||
else {
|
||||
return cublasCsyr2k(
|
||||
handle,
|
||||
uplo,
|
||||
trans_A,
|
||||
configuration.problem_size.n(),
|
||||
configuration.problem_size.k(),
|
||||
static_cast<const cuComplex*>(arguments.alpha),
|
||||
static_cast<const cuComplex*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const cuComplex*>(arguments.B),
|
||||
int(configuration.ldb),
|
||||
static_cast<const cuComplex*>(arguments.beta),
|
||||
static_cast<cuComplex*>(arguments.D),
|
||||
int(configuration.ldc)
|
||||
);
|
||||
}
|
||||
} else {
|
||||
return CUBLAS_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
}
|
||||
else {
|
||||
return CUBLAS_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Returns a status if cuBLAS can satisfy a particular TRMM description
|
||||
Status cublas_satisfies(library::TrmmDescription const &desc) {
|
||||
auto const &math_instruction = desc.tile_description.math_instruction;
|
||||
|
||||
if (math_instruction.element_accumulator == library::NumericTypeID::kS32 &&
|
||||
math_instruction.opcode_class == library::OpcodeClassID::kTensorOp) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
// output type S4 and S8 not supported in cuBLAS
|
||||
if (desc.D.element == library::NumericTypeID::kS4 ||
|
||||
desc.D.element == library::NumericTypeID::kS8) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
// input type BF16 and TF32 not supported in cuBLAS
|
||||
if (desc.A.element == library::NumericTypeID::kBF16 ||
|
||||
desc.A.element == library::NumericTypeID::kTF32) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace detail {
|
||||
|
||||
cublasTrmmDispatcher::cublasTrmmDispatcher(
|
||||
library::TrmmDescription const &op_desc,
|
||||
library::TrmmConfiguration configuration_,
|
||||
library::TrmmArguments arguments_
|
||||
):
|
||||
configuration(configuration_), arguments(arguments_), status(Status::kSuccess) {
|
||||
|
||||
bool good = true;
|
||||
|
||||
good = (good && get_cublas_transpose_operation(trans_A, op_desc.A.layout, op_desc.transform_A));
|
||||
good = (good && get_cublas_side_mode(side, op_desc.side_mode));
|
||||
good = (good && get_cublas_fill_mode(uplo, op_desc.fill_mode));
|
||||
good = (good && get_cublas_diag_type(diag, op_desc.diag_type));
|
||||
good = (good && get_cublas_datatype(data_type_A, op_desc.A.element));
|
||||
good = (good && get_cublas_datatype(data_type_B, op_desc.B.element));
|
||||
good = (good && get_cublas_datatype(data_type_D, op_desc.D.element));
|
||||
|
||||
// if A is Transposed, then for cuBLAS that is inverted Fill Mode.
|
||||
if (trans_A == CUBLAS_OP_T || trans_A == CUBLAS_OP_C) {
|
||||
if (uplo == CUBLAS_FILL_MODE_LOWER)
|
||||
uplo = CUBLAS_FILL_MODE_UPPER;
|
||||
else
|
||||
uplo = CUBLAS_FILL_MODE_LOWER;
|
||||
}
|
||||
|
||||
good = (good && get_cublas_datatype(
|
||||
compute_data_type,
|
||||
op_desc.tile_description.math_instruction.element_accumulator));
|
||||
|
||||
// cuBLAS introduces a separate cublasComputeType enumerant to more precisely describe
|
||||
// internal numerical data types used in the computation.
|
||||
#if (__CUDACC_VER_MAJOR__ >= 11)
|
||||
library::OpcodeClassID const & opcode_class =
|
||||
op_desc.tile_description.math_instruction.opcode_class;
|
||||
|
||||
if (good &&
|
||||
op_desc.A.element == library::NumericTypeID::kF32 &&
|
||||
opcode_class == library::OpcodeClassID::kTensorOp) {
|
||||
|
||||
compute_type = CUBLAS_COMPUTE_32F_FAST_TF32;
|
||||
}
|
||||
else if (good) {
|
||||
bool const isPedantic = false;
|
||||
switch (compute_data_type) {
|
||||
case CUDA_R_32F:
|
||||
case CUDA_C_32F:
|
||||
compute_type = isPedantic ? CUBLAS_COMPUTE_32F_PEDANTIC : CUBLAS_COMPUTE_32F;
|
||||
break;
|
||||
case CUDA_R_64F:
|
||||
case CUDA_C_64F:
|
||||
compute_type = isPedantic ? CUBLAS_COMPUTE_64F_PEDANTIC : CUBLAS_COMPUTE_64F;
|
||||
break;
|
||||
case CUDA_R_16F:
|
||||
compute_type = isPedantic ? CUBLAS_COMPUTE_16F_PEDANTIC : CUBLAS_COMPUTE_16F;
|
||||
break;
|
||||
case CUDA_R_32I:
|
||||
compute_type = isPedantic ? CUBLAS_COMPUTE_32I_PEDANTIC : CUBLAS_COMPUTE_32I;
|
||||
break;
|
||||
default:
|
||||
good = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
#endif // __CUDACC_VER_MAJOR__ >= 11
|
||||
|
||||
if (!good) {
|
||||
status = Status::kErrorNotSupported;
|
||||
}
|
||||
}
|
||||
|
||||
/// Executes TRMM using these arguments
|
||||
cublasStatus_t cublasTrmmDispatcher::operator()(cublasHandle_t handle) {
|
||||
|
||||
if (data_type_A == data_type_D && data_type_A == CUDA_R_64F) {
|
||||
return cublasDtrmm(
|
||||
handle,
|
||||
side,
|
||||
uplo,
|
||||
trans_A,
|
||||
diag,
|
||||
configuration.problem_size.m(),
|
||||
configuration.problem_size.n(),
|
||||
static_cast<const double*>(arguments.alpha),
|
||||
static_cast<const double*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const double*>(arguments.B),
|
||||
int(configuration.ldb),
|
||||
static_cast<double*>(arguments.D),
|
||||
int(configuration.ldd)
|
||||
);
|
||||
} else if (data_type_A == data_type_D && data_type_A == CUDA_R_32F) {
|
||||
|
||||
#if (__CUDACC_VER_MAJOR__ >= 11)
|
||||
if (cublasSetMathMode(handle, CUBLAS_TF32_TENSOR_OP_MATH) != CUBLAS_STATUS_SUCCESS)
|
||||
return CUBLAS_STATUS_NOT_SUPPORTED;
|
||||
#endif
|
||||
|
||||
return cublasStrmm(
|
||||
handle,
|
||||
side,
|
||||
uplo,
|
||||
trans_A,
|
||||
diag,
|
||||
configuration.problem_size.m(),
|
||||
configuration.problem_size.n(),
|
||||
static_cast<const float*>(arguments.alpha),
|
||||
static_cast<const float*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const float*>(arguments.B),
|
||||
int(configuration.ldb),
|
||||
static_cast<float*>(arguments.D),
|
||||
int(configuration.ldd)
|
||||
);
|
||||
} else if (data_type_A == data_type_D && data_type_A == CUDA_C_64F) {
|
||||
return cublasZtrmm(
|
||||
handle,
|
||||
side,
|
||||
uplo,
|
||||
trans_A,
|
||||
diag,
|
||||
configuration.problem_size.m(),
|
||||
configuration.problem_size.n(),
|
||||
static_cast<const cuDoubleComplex*>(arguments.alpha),
|
||||
static_cast<const cuDoubleComplex*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const cuDoubleComplex*>(arguments.B),
|
||||
int(configuration.ldb),
|
||||
static_cast<cuDoubleComplex*>(arguments.D),
|
||||
int(configuration.ldd)
|
||||
);
|
||||
} else if (data_type_A == data_type_D && data_type_A == CUDA_C_32F) {
|
||||
|
||||
#if (__CUDACC_VER_MAJOR__ >= 11)
|
||||
if (cublasSetMathMode(handle, CUBLAS_TF32_TENSOR_OP_MATH) != CUBLAS_STATUS_SUCCESS)
|
||||
return CUBLAS_STATUS_NOT_SUPPORTED;
|
||||
#endif
|
||||
|
||||
return cublasCtrmm(
|
||||
handle,
|
||||
side,
|
||||
uplo,
|
||||
trans_A,
|
||||
diag,
|
||||
configuration.problem_size.m(),
|
||||
configuration.problem_size.n(),
|
||||
static_cast<const cuComplex*>(arguments.alpha),
|
||||
static_cast<const cuComplex*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const cuComplex*>(arguments.B),
|
||||
int(configuration.ldb),
|
||||
static_cast<cuComplex*>(arguments.D),
|
||||
int(configuration.ldd)
|
||||
);
|
||||
} else {
|
||||
return CUBLAS_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Returns a status if cuBLAS can satisfy a particular Symm description
|
||||
Status cublas_satisfies(library::SymmDescription const &desc) {
|
||||
auto const &math_instruction = desc.tile_description.math_instruction;
|
||||
|
||||
if (math_instruction.element_accumulator == library::NumericTypeID::kS32 &&
|
||||
math_instruction.opcode_class == library::OpcodeClassID::kTensorOp) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
// output type S4 and S8 not supported in cuBLAS
|
||||
if (desc.C.element == library::NumericTypeID::kS4 ||
|
||||
desc.C.element == library::NumericTypeID::kS8) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
// input type BF16 and TF32 not supported in cuBLAS
|
||||
if (desc.A.element == library::NumericTypeID::kBF16 ||
|
||||
desc.A.element == library::NumericTypeID::kTF32) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
// input type BF16 and TF32 not supported in cuBLAS
|
||||
if (desc.B.element == library::NumericTypeID::kBF16 ||
|
||||
desc.B.element == library::NumericTypeID::kTF32) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
// only column major layout is supported in cuBLAS
|
||||
if (desc.A.layout != library::LayoutTypeID::kColumnMajor ||
|
||||
desc.transform_A != library::ComplexTransform::kNone) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace detail {
|
||||
|
||||
cublasSymmDispatcher::cublasSymmDispatcher(
|
||||
library::SymmDescription const &op_desc,
|
||||
library::SymmConfiguration configuration_,
|
||||
library::SymmArguments arguments_
|
||||
):
|
||||
configuration(configuration_), arguments(arguments_), status(Status::kSuccess) {
|
||||
|
||||
blas_mode = op_desc.blas_mode;
|
||||
|
||||
bool good = true;
|
||||
|
||||
good = (good && get_cublas_side_mode(side, op_desc.side_mode));
|
||||
good = (good && get_cublas_fill_mode(uplo, op_desc.fill_mode));
|
||||
good = (good && get_cublas_datatype(data_type_A, op_desc.A.element));
|
||||
good = (good && get_cublas_datatype(data_type_C, op_desc.C.element));
|
||||
|
||||
good = (good && get_cublas_datatype(
|
||||
compute_data_type,
|
||||
op_desc.tile_description.math_instruction.element_accumulator));
|
||||
|
||||
// cuBLAS introduces a separate cublasComputeType enumerant to more precisely describe
|
||||
// internal numerical data types used in the computation.
|
||||
#if (__CUDACC_VER_MAJOR__ >= 11)
|
||||
library::OpcodeClassID const & opcode_class =
|
||||
op_desc.tile_description.math_instruction.opcode_class;
|
||||
|
||||
if (good &&
|
||||
op_desc.A.element == library::NumericTypeID::kF32 &&
|
||||
opcode_class == library::OpcodeClassID::kTensorOp) {
|
||||
|
||||
compute_type = CUBLAS_COMPUTE_32F_FAST_TF32;
|
||||
}
|
||||
else if (good) {
|
||||
bool const isPedantic = false;
|
||||
switch (compute_data_type) {
|
||||
case CUDA_R_32F:
|
||||
case CUDA_C_32F:
|
||||
compute_type = isPedantic ? CUBLAS_COMPUTE_32F_PEDANTIC : CUBLAS_COMPUTE_32F;
|
||||
break;
|
||||
case CUDA_R_64F:
|
||||
case CUDA_C_64F:
|
||||
compute_type = isPedantic ? CUBLAS_COMPUTE_64F_PEDANTIC : CUBLAS_COMPUTE_64F;
|
||||
break;
|
||||
case CUDA_R_16F:
|
||||
compute_type = isPedantic ? CUBLAS_COMPUTE_16F_PEDANTIC : CUBLAS_COMPUTE_16F;
|
||||
break;
|
||||
case CUDA_R_32I:
|
||||
compute_type = isPedantic ? CUBLAS_COMPUTE_32I_PEDANTIC : CUBLAS_COMPUTE_32I;
|
||||
break;
|
||||
default:
|
||||
good = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
#endif // __CUDACC_VER_MAJOR__ >= 11
|
||||
|
||||
if (!good) {
|
||||
status = Status::kErrorNotSupported;
|
||||
}
|
||||
}
|
||||
|
||||
/// Executes Symm using these arguments
|
||||
cublasStatus_t cublasSymmDispatcher::operator()(cublasHandle_t handle) {
|
||||
|
||||
// SYMM and HEMM
|
||||
if (data_type_A == data_type_C && data_type_A == CUDA_R_64F) {
|
||||
return cublasDsymm(
|
||||
handle,
|
||||
side,
|
||||
uplo,
|
||||
configuration.problem_size.m(),
|
||||
configuration.problem_size.n(),
|
||||
static_cast<const double*>(arguments.alpha),
|
||||
static_cast<const double*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const double*>(arguments.B),
|
||||
int(configuration.ldb),
|
||||
static_cast<const double*>(arguments.beta),
|
||||
static_cast<double*>(arguments.D),
|
||||
int(configuration.ldc)
|
||||
);
|
||||
} else if (data_type_A == data_type_C && data_type_A == CUDA_R_32F) {
|
||||
|
||||
#if (__CUDACC_VER_MAJOR__ >= 11)
|
||||
if (cublasSetMathMode(handle, CUBLAS_TF32_TENSOR_OP_MATH) != CUBLAS_STATUS_SUCCESS)
|
||||
return CUBLAS_STATUS_NOT_SUPPORTED;
|
||||
#endif
|
||||
|
||||
return cublasSsymm(
|
||||
handle,
|
||||
side,
|
||||
uplo,
|
||||
configuration.problem_size.m(),
|
||||
configuration.problem_size.n(),
|
||||
static_cast<const float*>(arguments.alpha),
|
||||
static_cast<const float*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const float*>(arguments.B),
|
||||
int(configuration.ldb),
|
||||
static_cast<const float*>(arguments.beta),
|
||||
static_cast<float*>(arguments.D),
|
||||
int(configuration.ldc)
|
||||
);
|
||||
} else if (data_type_A == data_type_C && data_type_A == CUDA_C_64F) {
|
||||
|
||||
if (blas_mode == BlasMode::kHermitian) {
|
||||
return cublasZhemm(
|
||||
handle,
|
||||
side,
|
||||
uplo,
|
||||
configuration.problem_size.m(),
|
||||
configuration.problem_size.n(),
|
||||
static_cast<const cuDoubleComplex*>(arguments.alpha),
|
||||
static_cast<const cuDoubleComplex*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const cuDoubleComplex*>(arguments.B),
|
||||
int(configuration.ldb),
|
||||
static_cast<const cuDoubleComplex*>(arguments.beta),
|
||||
static_cast<cuDoubleComplex*>(arguments.D),
|
||||
int(configuration.ldc)
|
||||
);
|
||||
}
|
||||
else {
|
||||
return cublasZsymm(
|
||||
handle,
|
||||
side,
|
||||
uplo,
|
||||
configuration.problem_size.m(),
|
||||
configuration.problem_size.n(),
|
||||
static_cast<const cuDoubleComplex*>(arguments.alpha),
|
||||
static_cast<const cuDoubleComplex*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const cuDoubleComplex*>(arguments.B),
|
||||
int(configuration.ldb),
|
||||
static_cast<const cuDoubleComplex*>(arguments.beta),
|
||||
static_cast<cuDoubleComplex*>(arguments.D),
|
||||
int(configuration.ldc)
|
||||
);
|
||||
}
|
||||
|
||||
} else if (data_type_A == data_type_C && data_type_A == CUDA_C_32F) {
|
||||
|
||||
#if (__CUDACC_VER_MAJOR__ >= 11)
|
||||
if (cublasSetMathMode(handle, CUBLAS_TF32_TENSOR_OP_MATH) != CUBLAS_STATUS_SUCCESS)
|
||||
return CUBLAS_STATUS_NOT_SUPPORTED;
|
||||
#endif
|
||||
|
||||
if (blas_mode == BlasMode::kHermitian) {
|
||||
return cublasChemm(
|
||||
handle,
|
||||
side,
|
||||
uplo,
|
||||
configuration.problem_size.m(),
|
||||
configuration.problem_size.n(),
|
||||
static_cast<const cuComplex*>(arguments.alpha),
|
||||
static_cast<const cuComplex*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const cuComplex*>(arguments.B),
|
||||
int(configuration.ldb),
|
||||
static_cast<const cuComplex*>(arguments.beta),
|
||||
static_cast<cuComplex*>(arguments.D),
|
||||
int(configuration.ldc)
|
||||
);
|
||||
}
|
||||
else {
|
||||
return cublasCsymm(
|
||||
handle,
|
||||
side,
|
||||
uplo,
|
||||
configuration.problem_size.m(),
|
||||
configuration.problem_size.n(),
|
||||
static_cast<const cuComplex*>(arguments.alpha),
|
||||
static_cast<const cuComplex*>(arguments.A),
|
||||
int(configuration.lda),
|
||||
static_cast<const cuComplex*>(arguments.B),
|
||||
int(configuration.ldb),
|
||||
static_cast<const cuComplex*>(arguments.beta),
|
||||
static_cast<cuComplex*>(arguments.D),
|
||||
int(configuration.ldc)
|
||||
);
|
||||
}
|
||||
} else {
|
||||
return CUBLAS_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
@@ -34,6 +40,8 @@
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/util.h"
|
||||
#include "cutlass/blas3.h"
|
||||
|
||||
#include "options.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -68,6 +76,15 @@ cublasGemmAlgo_t get_cublas_gemm_algo(
|
||||
/// Returns a status if cuBLAS can satisfy a particular GEMM description
|
||||
Status cublas_satisfies(library::GemmDescription const &desc);
|
||||
|
||||
/// Returns a status if cuBLAS can satisfy a particular RankK description
|
||||
Status cublas_satisfies(library::RankKDescription const &desc);
|
||||
|
||||
/// Returns a status if cuBLAS can satisfy a particular TRMM description
|
||||
Status cublas_satisfies(library::TrmmDescription const &desc);
|
||||
|
||||
/// Returns a status if cuBLAS can satisfy a particular SYMM/HEMM description
|
||||
Status cublas_satisfies(library::SymmDescription const &desc);
|
||||
|
||||
/// This is a helper class to create cublasHandle_t automatically on CublasCreate object creation and
|
||||
/// to destroy cublasHandle_t on CublasCreate object destruction.
|
||||
/// Additionaly, it provides implicit cast from CublasCreate's object to cublasHandle_t's object
|
||||
@@ -211,6 +228,127 @@ struct cublasGemmExDispatcher {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Dispatcher to cublas rank k update kernels
|
||||
struct cublasRankKDispatcher {
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
library::RankKConfiguration configuration;
|
||||
library::RankKArguments arguments;
|
||||
|
||||
// cublass-specific data structures to fill cublas API call arguments
|
||||
cublasOperation_t trans_A;
|
||||
cublasFillMode_t uplo;
|
||||
cudaDataType_t data_type_A;
|
||||
cudaDataType_t data_type_C;
|
||||
cudaDataType_t compute_data_type;
|
||||
|
||||
#if (__CUDACC_VER_MAJOR__ >= 11)
|
||||
cublasComputeType_t compute_type;
|
||||
#endif
|
||||
|
||||
int num_ranks; //(rank-k or rank-2k)
|
||||
BlasMode blas_mode; //(symmetric or hermitian)
|
||||
Status status;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
cublasRankKDispatcher(
|
||||
library::RankKDescription const &op_desc,
|
||||
library::RankKConfiguration configuration_,
|
||||
library::RankKArguments arguments_
|
||||
);
|
||||
|
||||
/// Executes RankK using these arguments
|
||||
cublasStatus_t operator()(cublasHandle_t handle);
|
||||
};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Dispatcher to cublasTrmm()
|
||||
struct cublasTrmmDispatcher {
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
library::TrmmConfiguration configuration;
|
||||
library::TrmmArguments arguments;
|
||||
|
||||
// cublass-specific data structures to fill cublas API call arguments
|
||||
cublasOperation_t trans_A;
|
||||
cublasSideMode_t side;
|
||||
cublasFillMode_t uplo;
|
||||
cublasDiagType_t diag;
|
||||
cudaDataType_t data_type_A;
|
||||
cudaDataType_t data_type_B;
|
||||
cudaDataType_t data_type_D;
|
||||
cudaDataType_t compute_data_type;
|
||||
|
||||
#if (__CUDACC_VER_MAJOR__ >= 11)
|
||||
cublasComputeType_t compute_type;
|
||||
#endif
|
||||
|
||||
Status status;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
cublasTrmmDispatcher(
|
||||
library::TrmmDescription const &op_desc,
|
||||
library::TrmmConfiguration configuration_,
|
||||
library::TrmmArguments arguments_
|
||||
);
|
||||
|
||||
/// Executes TRMM using these arguments
|
||||
cublasStatus_t operator()(cublasHandle_t handle);
|
||||
};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Dispatcher to cublas symm/hemm update kernels
|
||||
struct cublasSymmDispatcher {
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
library::SymmConfiguration configuration;
|
||||
library::SymmArguments arguments;
|
||||
|
||||
// cublass-specific data structures to fill cublas API call arguments
|
||||
cublasSideMode_t side;
|
||||
cublasFillMode_t uplo;
|
||||
cudaDataType_t data_type_A;
|
||||
cudaDataType_t data_type_B;
|
||||
cudaDataType_t data_type_C;
|
||||
cudaDataType_t compute_data_type;
|
||||
|
||||
#if (__CUDACC_VER_MAJOR__ >= 11)
|
||||
cublasComputeType_t compute_type;
|
||||
#endif
|
||||
|
||||
BlasMode blas_mode; //(symmetric or hermitian)
|
||||
Status status;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
cublasSymmDispatcher(
|
||||
library::SymmDescription const &op_desc,
|
||||
library::SymmConfiguration configuration_,
|
||||
library::SymmArguments arguments_
|
||||
);
|
||||
|
||||
/// Executes Symm using these arguments
|
||||
cublasStatus_t operator()(cublasHandle_t handle);
|
||||
};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace detail
|
||||
|
||||
} // namespace profiler
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
@@ -32,6 +38,10 @@
|
||||
// Profiler includes
|
||||
#include "cutlass_profiler.h"
|
||||
#include "gemm_operation_profiler.h"
|
||||
#include "rank_k_operation_profiler.h"
|
||||
#include "rank_2k_operation_profiler.h"
|
||||
#include "trmm_operation_profiler.h"
|
||||
#include "symm_operation_profiler.h"
|
||||
#include "conv2d_operation_profiler.h"
|
||||
#include "conv3d_operation_profiler.h"
|
||||
#include "sparse_gemm_operation_profiler.h"
|
||||
@@ -56,6 +66,13 @@ CutlassProfiler::CutlassProfiler(
|
||||
|
||||
operation_profilers_.emplace_back(new Conv3dOperationProfiler(options));
|
||||
|
||||
operation_profilers_.emplace_back(new RankKOperationProfiler(options));
|
||||
|
||||
operation_profilers_.emplace_back(new Rank2KOperationProfiler(options));
|
||||
|
||||
operation_profilers_.emplace_back(new TrmmOperationProfiler(options));
|
||||
|
||||
operation_profilers_.emplace_back(new SymmOperationProfiler(options));
|
||||
}
|
||||
|
||||
CutlassProfiler::~CutlassProfiler() {
|
||||
@@ -174,6 +191,9 @@ void CutlassProfiler::print_usage_(std::ostream &out) {
|
||||
|
||||
out << "\n\nFor details about a particular function, specify the function name with --help.\n\nExample:\n\n"
|
||||
<< " $ cutlass_profiler --operation=Gemm --help\n\n"
|
||||
<< " $ cutlass_profiler --operation=RankK --help\n\n"
|
||||
<< " $ cutlass_profiler --operation=Trmm --help\n\n"
|
||||
<< " $ cutlass_profiler --operation=Symm --help\n\n"
|
||||
<< " $ cutlass_profiler --operation=Conv3d --help\n\n"
|
||||
<< " $ cutlass_profiler --operation=Conv2d --help\n\n"
|
||||
<< " $ cutlass_profiler --operation=SparseGemm --help\n\n"
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
+24
-18
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
+24
-18
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -0,0 +1,727 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Execution environment
|
||||
|
||||
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
#include <iomanip>
|
||||
#include <ios>
|
||||
|
||||
#include "cutlass/core_io.h"
|
||||
|
||||
#include "cublas_helpers.h"
|
||||
#include "rank_2k_operation_profiler.h"
|
||||
#include "gpu_timer.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Ctor
|
||||
Rank2KOperationProfiler::Rank2KOperationProfiler(Options const &options):
|
||||
OperationProfiler(
|
||||
options,
|
||||
library::OperationKind::kRank2K,
|
||||
{
|
||||
{ArgumentTypeID::kEnumerated, {"rank_k_kind"}, "Variant of RankK (universal)"},
|
||||
{ArgumentTypeID::kInteger, {"n", "problem-size::n"}, "N dimension of the RankK problem space"},
|
||||
{ArgumentTypeID::kInteger, {"k", "problem-size::k"}, "K dimension of the RankK problem space"},
|
||||
{ArgumentTypeID::kTensor, {"A"}, "Tensor storing the A operand"},
|
||||
{ArgumentTypeID::kTensor, {"B"}, "Tensor storing the B operand"},
|
||||
{ArgumentTypeID::kTensor, {"C"}, "Tensor storing the C operand"},
|
||||
{ArgumentTypeID::kEnumerated, {"fill_mode"}, "Fill Mode for RankK kernel (lower or upper)"},
|
||||
{ArgumentTypeID::kEnumerated, {"blas_mode"}, "Blas Mode for RankK kernel (symmetric or hermitian)"},
|
||||
{ArgumentTypeID::kScalar, {"alpha", "epilogue::alpha"}, "Epilogue scalar alpha"},
|
||||
{ArgumentTypeID::kScalar, {"beta", "epilogue::beta"}, "Epilogue scalar beta"},
|
||||
{ArgumentTypeID::kInteger, {"split_k_slices", "split-k-slices"}, "Number of partitions of K dimension"},
|
||||
{ArgumentTypeID::kInteger, {"batch_count", "batch-count"}, "Number of RankK computed in one batch"},
|
||||
},
|
||||
{ library::Provider::kCUBLAS}
|
||||
) {
|
||||
description_ = " Rank 2k Update. D = alpha * (A*B^T + B*A^T) + beta * C (symmetric) or D = alpha * (A*B^H+B*A^H) + beta * C (hermitian)";
|
||||
}
|
||||
|
||||
/// Destructor
|
||||
Rank2KOperationProfiler::~Rank2KOperationProfiler() {
|
||||
|
||||
}
|
||||
|
||||
/// Prints usage statement for the math function
|
||||
void Rank2KOperationProfiler::print_usage(std::ostream &out) const {
|
||||
out << "RankK" << "\n\n";
|
||||
|
||||
OperationProfiler::print_usage(out);
|
||||
}
|
||||
|
||||
/// Prints examples
|
||||
void Rank2KOperationProfiler::print_examples(std::ostream &out) const {
|
||||
|
||||
out << "\nExamples:\n\n"
|
||||
<< "Profile a particular problem size Syrk kernel:\n"
|
||||
<< " $ cutlass_profiler --operation=rank_2k --blas_mode=symmetric --n=1024 --k=128\n\n"
|
||||
|
||||
<< "Profile a particular problem size Herk kernel:\n"
|
||||
<< " $ cutlass_profiler --operation=rank_2k --blas_mode=hermitian --n=1024 --k=128\n\n"
|
||||
|
||||
<< "Schmoo over problem size and beta:\n"
|
||||
<< " $ cutlass_profiler --operation=rank_2k --n=1024:4096:256 --k=128:8192:128 --beta=0,1,2.5\n\n"
|
||||
|
||||
<< "Schmoo over accumulator types:\n"
|
||||
<< " $ cutlass_profiler --operation=rank_2k --accumulator-type=f16,f32\n\n"
|
||||
|
||||
<< "Schmoo over fill modees:\n"
|
||||
<< " $ cutlass_profiler --operation=rank_2k --fill_mode=lower/upper\n\n"
|
||||
|
||||
<< "Run when A is f16 with column-major or A is any datatype with row-major (For column major, use column, col, or n. For row major use, row or t):\n"
|
||||
<< " $ cutlass_profiler --operation=rank_2k --A=f16:column or --A=*:row\n\n"
|
||||
|
||||
<< "Using various input value distribution:\n"
|
||||
<< " $ cutlass_profiler --operation=rank_2k --dist=uniform,min:0,max:3\n"
|
||||
<< " $ cutlass_profiler --operation=rank_2k --dist=gaussian,mean:0,stddev:3\n"
|
||||
<< " $ cutlass_profiler --operation=rank_2k --dist=sequential,start:0,delta:1\n\n"
|
||||
|
||||
<< "Run a kernel with cta tile size of 256x128x32 and save workspace if results are incorrect (note that --cta-tile::k=32 is default cta-tile size):\n"
|
||||
<< " $ cutlass_profiler --operation=rank_2k --cta_m=256 --cta_n=128 --cta_k=32 --save-workspace=incorrect\n\n"
|
||||
|
||||
<< "Test your changes to rank_2k kernels with a quick functional test and save results in functional-test.csv:\n"
|
||||
<< " $ cutlass_profiler --operation=rank_2k \\ \n"
|
||||
<< " --n=8,56,120,136,256,264,512,520,1024,1032,4096,8192,16384 \\ \n"
|
||||
<< " --k=8,16,32,64,128,256,288,384,504,512,520 \\ \n"
|
||||
<< " --beta=0,1,2 --profiling-iterations=1 \\ \n"
|
||||
<< " --providers=cutlass --output=functional-test.csv\n\n";
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if 0
|
||||
// used this for debugging
|
||||
static std::string byte_string(std::vector<uint8_t> const &bytes) {
|
||||
std::stringstream ss;
|
||||
|
||||
ss << "0x";
|
||||
|
||||
for (size_t idx = bytes.size(); idx > 0; --idx) {
|
||||
ss << std::hex << std::setw(2) << std::setfill('0') << uint32_t(bytes.at(idx - 1));
|
||||
}
|
||||
|
||||
return ss.str();
|
||||
}
|
||||
#endif
|
||||
|
||||
Status Rank2KOperationProfiler::RankKProblem::parse(
|
||||
library::RankKDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
if (!arg_as_int(this->n, "n", problem_space, problem)) {
|
||||
// default value
|
||||
this->n = 1024;
|
||||
}
|
||||
|
||||
if (!arg_as_int(this->k, "k", problem_space, problem)) {
|
||||
// default value
|
||||
this->k = 1024;
|
||||
}
|
||||
|
||||
if (!arg_as_int(this->split_k_slices, "split_k_slices", problem_space, problem)) {
|
||||
// default value
|
||||
this->split_k_slices = 1;
|
||||
}
|
||||
|
||||
if (!arg_as_int(this->batch_count, "batch_count", problem_space, problem)) {
|
||||
// default value
|
||||
this->batch_count = 1;
|
||||
}
|
||||
|
||||
if (this->split_k_slices > 1 && this->batch_count > 1) {
|
||||
// At least one of these must be one
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!tensor_description_satisfies(operation_desc.A, "A", problem_space, problem)) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!tensor_description_satisfies(operation_desc.B, "B", problem_space, problem)) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!tensor_description_satisfies(operation_desc.C, "C", problem_space, problem)) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!arg_as_scalar(
|
||||
this->alpha,
|
||||
operation_desc.element_epilogue,
|
||||
"alpha",
|
||||
problem_space,
|
||||
problem)) {
|
||||
|
||||
if (!cast_from_double(this->alpha, operation_desc.element_epilogue, 1)) {
|
||||
return Status::kErrorInternal;
|
||||
}
|
||||
}
|
||||
|
||||
if (!arg_as_scalar(
|
||||
this->beta,
|
||||
operation_desc.element_epilogue,
|
||||
"beta",
|
||||
problem_space,
|
||||
problem)) {
|
||||
|
||||
if (!cast_from_double(this->beta, operation_desc.element_epilogue, 0)) {
|
||||
return Status::kErrorInternal;
|
||||
}
|
||||
}
|
||||
|
||||
this->lda = DeviceAllocation::get_packed_layout(
|
||||
operation_desc.A.layout, {int(this->n), int(this->k)}).front();
|
||||
|
||||
this->ldb = DeviceAllocation::get_packed_layout(
|
||||
operation_desc.B.layout, {int(this->n), int(this->k)}).front();
|
||||
|
||||
this->ldc = DeviceAllocation::get_packed_layout(
|
||||
operation_desc.C.layout, {int(this->n), int(this->n)}).front();
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/// Total number of bytes loaded
|
||||
int64_t Rank2KOperationProfiler::RankKProblem::bytes(library::RankKDescription const &operation_desc) const {
|
||||
// Input bytes read and Output bytes written for the gemm problem
|
||||
int64_t bytes =
|
||||
2 * int64_t(library::sizeof_bits(operation_desc.A.element) * n / 8) * k +
|
||||
2 * int64_t(library::sizeof_bits(operation_desc.B.element) * n / 8) * k +
|
||||
// Half matrix including the diagonal will have (N*(N+1))/2 elements
|
||||
int64_t(library::sizeof_bits(operation_desc.C.element) * n / 8) * (n+1) / 2;
|
||||
|
||||
// Set is_beta_zero true if beta is zero
|
||||
bool is_beta_zero = std::all_of(beta.begin(), beta.end(), [](uint8_t i) { return i==0; });
|
||||
|
||||
// Output bytes read for the gemm problem for non-zero beta values
|
||||
if (!is_beta_zero) {
|
||||
bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * n / 8) * (n+1) / 2;
|
||||
}
|
||||
|
||||
bytes *= batch_count;
|
||||
|
||||
return bytes;
|
||||
}
|
||||
|
||||
/// Total number of flops computed
|
||||
int64_t Rank2KOperationProfiler::RankKProblem::flops(library::RankKDescription const &operation_desc) const {
|
||||
|
||||
// FLOPs = 2 * n(n+1)k/2 [mma1] + 2 * n(n+1)k/2 [mma2] + 2 * n(n+1)/2 [epilogue]
|
||||
// FLOPs = n(n+1)(2k + 1)
|
||||
int64_t flops_ = n * (n + 1) * (2*k + 1);
|
||||
|
||||
// complex-valued support
|
||||
switch (operation_desc.tile_description.math_instruction.math_operation) {
|
||||
case library::MathOperationID::kMultiplyAddComplex:
|
||||
flops_ *= 4;
|
||||
break;
|
||||
|
||||
case library::MathOperationID::kMultiplyAddComplexFastF32:
|
||||
flops_ *= 4;
|
||||
break;
|
||||
|
||||
case library::MathOperationID::kMultiplyAddGaussianComplex:
|
||||
flops_ *= 3;
|
||||
break;
|
||||
|
||||
default: break;
|
||||
}
|
||||
|
||||
return flops_;
|
||||
}
|
||||
|
||||
/// Initializes a performance result
|
||||
void Rank2KOperationProfiler::RankKProblem::initialize_result(
|
||||
PerformanceResult &result,
|
||||
library::RankKDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space) {
|
||||
|
||||
result.arguments.resize(problem_space.rank());
|
||||
|
||||
set_argument(result, "rank_k_kind", problem_space, library::to_string(operation_desc.rank_k_kind));
|
||||
|
||||
set_argument(result, "A", problem_space,
|
||||
std::string(library::to_string(operation_desc.A.element)) + ":" + library::to_string(operation_desc.A.layout));
|
||||
|
||||
set_argument(result, "B", problem_space,
|
||||
std::string(library::to_string(operation_desc.B.element)) + ":" + library::to_string(operation_desc.B.layout));
|
||||
|
||||
set_argument(result, "C", problem_space,
|
||||
std::string(library::to_string(operation_desc.C.element)) + ":" + library::to_string(operation_desc.C.layout));
|
||||
|
||||
set_argument(result, "fill_mode", problem_space, library::to_string(operation_desc.fill_mode));
|
||||
|
||||
set_argument(result, "blas_mode", problem_space, library::to_string(operation_desc.blas_mode));
|
||||
|
||||
set_argument(result, "n", problem_space, n);
|
||||
set_argument(result, "k", problem_space, k);
|
||||
|
||||
set_argument(result, "split_k_slices", problem_space, split_k_slices);
|
||||
set_argument(result, "batch_count", problem_space, batch_count);
|
||||
|
||||
set_argument(result, "alpha", problem_space,
|
||||
library::lexical_cast(alpha, operation_desc.element_epilogue));
|
||||
|
||||
set_argument(result, "beta", problem_space,
|
||||
library::lexical_cast(beta, operation_desc.element_epilogue));
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Extracts the problem dimensions
|
||||
Status Rank2KOperationProfiler::initialize_configuration(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
library::RankKDescription const &operation_desc =
|
||||
static_cast<library::RankKDescription const &>(operation->description());
|
||||
|
||||
if (operation_desc.rank_k_kind != library::RankKKind::kUniversal) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
Status status = problem_.parse(operation_desc, problem_space, problem);
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
rank_k_workspace_.configuration.problem_size.m() = int(problem_.n);
|
||||
rank_k_workspace_.configuration.problem_size.n() = int(problem_.n);
|
||||
rank_k_workspace_.configuration.problem_size.k() = int(problem_.k);
|
||||
rank_k_workspace_.configuration.lda = problem_.lda;
|
||||
rank_k_workspace_.configuration.ldb = problem_.ldb;
|
||||
rank_k_workspace_.configuration.ldc = problem_.ldc;
|
||||
rank_k_workspace_.configuration.ldd = problem_.ldc;
|
||||
//rank_k_workspace_.configuration.split_k_slices = int(problem_.split_k_slices);
|
||||
rank_k_workspace_.configuration.batch_count = int(problem_.split_k_slices);
|
||||
|
||||
rank_k_workspace_.arguments.A = nullptr;
|
||||
rank_k_workspace_.arguments.B = nullptr;
|
||||
rank_k_workspace_.arguments.C = nullptr;
|
||||
rank_k_workspace_.arguments.D = nullptr;
|
||||
rank_k_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
rank_k_workspace_.arguments.beta = problem_.beta.data();
|
||||
rank_k_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
initialize_result_(this->model_result_, options, operation_desc, problem_space);
|
||||
|
||||
return operation->can_implement(&rank_k_workspace_.configuration, &rank_k_workspace_.arguments);
|
||||
}
|
||||
|
||||
/// Initializes the performance result
|
||||
void Rank2KOperationProfiler::initialize_result_(
|
||||
PerformanceResult &result,
|
||||
Options const &options,
|
||||
library::RankKDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space) {
|
||||
|
||||
result.provider = library::Provider::kCUTLASS;
|
||||
result.disposition = Disposition::kNotRun;
|
||||
result.status = Status::kSuccess;
|
||||
result.operation_name = operation_desc.name;
|
||||
|
||||
problem_.initialize_result(result, operation_desc, problem_space);
|
||||
|
||||
OperationProfiler::initialize_result_(result, operation_desc, problem_space);
|
||||
|
||||
|
||||
result.bytes = problem_.bytes(operation_desc);
|
||||
result.flops = problem_.flops(operation_desc);
|
||||
result.runtime = 0;
|
||||
|
||||
|
||||
}
|
||||
|
||||
/// Initializes workspace
|
||||
Status Rank2KOperationProfiler::initialize_workspace(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
library::RankKDescription const &operation_desc =
|
||||
static_cast<library::RankKDescription const &>(operation->description());
|
||||
|
||||
if (options.execution_mode != ExecutionMode::kDryRun) {
|
||||
|
||||
rank_k_workspace_.A = device_context.allocate_tensor(
|
||||
options,
|
||||
"A",
|
||||
operation_desc.A.element,
|
||||
operation_desc.A.layout,
|
||||
{int(problem_.n), int(problem_.k)},
|
||||
{int(problem_.lda)}
|
||||
);
|
||||
|
||||
rank_k_workspace_.B = device_context.allocate_tensor(
|
||||
options,
|
||||
"B",
|
||||
operation_desc.B.element,
|
||||
operation_desc.B.layout,
|
||||
{int(problem_.n), int(problem_.k)},
|
||||
{int(problem_.ldb)}
|
||||
);
|
||||
|
||||
rank_k_workspace_.C = device_context.allocate_tensor(
|
||||
options,
|
||||
"C",
|
||||
operation_desc.C.element,
|
||||
operation_desc.C.layout,
|
||||
{int(problem_.n), int(problem_.n)},
|
||||
{int(problem_.ldc)},
|
||||
1 // batch_count = 1, default
|
||||
);
|
||||
|
||||
rank_k_workspace_.Computed = device_context.allocate_tensor(
|
||||
"D",
|
||||
operation_desc.C.element,
|
||||
operation_desc.C.layout,
|
||||
{int(problem_.n), int(problem_.n)},
|
||||
{int(problem_.ldc)}
|
||||
);
|
||||
|
||||
rank_k_workspace_.Reference = device_context.allocate_tensor(
|
||||
"Reference",
|
||||
operation_desc.C.element,
|
||||
operation_desc.C.layout,
|
||||
{int(problem_.n), int(problem_.n)},
|
||||
{int(problem_.ldc)}
|
||||
);
|
||||
|
||||
rank_k_workspace_.Computed->copy_from_device(rank_k_workspace_.C->data());
|
||||
rank_k_workspace_.Reference->copy_from_device(rank_k_workspace_.C->data());
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// Initialize the CUTLASS operation
|
||||
//
|
||||
Status status = Status::kSuccess;
|
||||
|
||||
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
|
||||
|
||||
if (options.execution_mode != ExecutionMode::kDryRun) {
|
||||
|
||||
uint64_t workspace_size = operation->get_host_workspace_size(&rank_k_workspace_.configuration);
|
||||
rank_k_workspace_.host_workspace.resize(workspace_size, 0);
|
||||
|
||||
workspace_size = operation->get_device_workspace_size(&rank_k_workspace_.configuration);
|
||||
rank_k_workspace_.device_workspace.reset(library::NumericTypeID::kU8, workspace_size);
|
||||
|
||||
status = operation->initialize(
|
||||
&rank_k_workspace_.configuration,
|
||||
rank_k_workspace_.host_workspace.data(),
|
||||
rank_k_workspace_.device_workspace.data());
|
||||
}
|
||||
|
||||
//
|
||||
// If CUTLASS is enabled, generate a result for it
|
||||
//
|
||||
results_.push_back(model_result_);
|
||||
results_.back().provider = library::Provider::kCUTLASS;
|
||||
results_.back().op_kind = library::OperationKind::kRank2K;
|
||||
results_.back().disposition = Disposition::kNotRun;
|
||||
|
||||
for(auto provider : verification_providers_) {
|
||||
results_.back().verification_map[provider] = Disposition::kNotRun;
|
||||
}
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
bool Rank2KOperationProfiler::verify_cutlass(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
if (!options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (options.execution_mode == ExecutionMode::kDryRun) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Initialize structure containing RankK arguments
|
||||
rank_k_workspace_.arguments.A = rank_k_workspace_.A->data();
|
||||
rank_k_workspace_.arguments.B = rank_k_workspace_.B->data();
|
||||
rank_k_workspace_.arguments.C = rank_k_workspace_.C->data();
|
||||
rank_k_workspace_.arguments.D = rank_k_workspace_.Computed->data();
|
||||
rank_k_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
rank_k_workspace_.arguments.beta = problem_.beta.data();
|
||||
rank_k_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
//
|
||||
// Run the CUTLASS operation
|
||||
//
|
||||
|
||||
results_.back().status = operation->run(
|
||||
&rank_k_workspace_.arguments,
|
||||
rank_k_workspace_.host_workspace.data(),
|
||||
rank_k_workspace_.device_workspace.data());
|
||||
|
||||
if (results_.back().status != Status::kSuccess) {
|
||||
results_.back().disposition = Disposition::kFailed;
|
||||
return false;
|
||||
}
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
if (result != cudaSuccess) {
|
||||
results_.back().disposition = Disposition::kFailed;
|
||||
return false;
|
||||
}
|
||||
|
||||
// CUTLASS op ran the but not yet verified against any verification provider
|
||||
results_.back().disposition = Disposition::kNotVerified;
|
||||
|
||||
//
|
||||
// Run verification providers
|
||||
//
|
||||
|
||||
if (options.verification.enabled) {
|
||||
|
||||
#if CUTLASS_ENABLE_CUBLAS
|
||||
if (options.verification.provider_enabled(library::Provider::kCUBLAS)) {
|
||||
|
||||
// Guard against unsupported cases
|
||||
auto const & rank_k_desc = static_cast<library::RankKDescription const &>(operation->description());
|
||||
|
||||
if (cublas_satisfies(rank_k_desc) == Status::kSuccess) {
|
||||
|
||||
// call cublas verification if supported
|
||||
verify_with_cublas_(
|
||||
options,
|
||||
report,
|
||||
device_context,
|
||||
operation,
|
||||
problem_space,
|
||||
problem);
|
||||
}
|
||||
|
||||
else {
|
||||
// set verification map for cublas to not supported
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kNotSupported;
|
||||
}
|
||||
}
|
||||
#endif // #if CUTLASS_ENABLE_CUBLAS
|
||||
|
||||
// Update disposition to worst case verification outcome among all
|
||||
// verification providers which are supported
|
||||
bool is_any_verification_run_passed = false;
|
||||
for(auto &m : results_.back().verification_map) {
|
||||
if(m.second == Disposition::kFailed || m.second == Disposition::kIncorrect) {
|
||||
results_.back().disposition = m.second;
|
||||
return true;
|
||||
}
|
||||
if(!is_any_verification_run_passed && m.second == Disposition::kPassed) {
|
||||
is_any_verification_run_passed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if(is_any_verification_run_passed) {
|
||||
results_.back().disposition = Disposition::kPassed;
|
||||
}
|
||||
}
|
||||
|
||||
// Return true means continue profiling
|
||||
return true;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
bool Rank2KOperationProfiler::verify_with_cublas_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
|
||||
#if CUTLASS_ENABLE_CUBLAS
|
||||
|
||||
library::RankKDescription const &rank_k_desc =
|
||||
static_cast<library::RankKDescription const &>(operation->description());
|
||||
|
||||
//
|
||||
// Construct cuBLAS operators
|
||||
//
|
||||
|
||||
CublasCreate handle;
|
||||
cublasStatus_t status = handle.get_cublas_create_status();
|
||||
|
||||
if (status != CUBLAS_STATUS_SUCCESS) {
|
||||
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// Initialize state
|
||||
//
|
||||
|
||||
try {
|
||||
|
||||
//
|
||||
// Construct dispatcher to cublas<t>Syr2k()
|
||||
//
|
||||
|
||||
// Initialize structure containing RankK arguments
|
||||
rank_k_workspace_.arguments.A = rank_k_workspace_.A->data();
|
||||
rank_k_workspace_.arguments.B = rank_k_workspace_.B->data();
|
||||
rank_k_workspace_.arguments.C = rank_k_workspace_.Reference->data();
|
||||
rank_k_workspace_.arguments.D = rank_k_workspace_.Reference->data();
|
||||
rank_k_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
rank_k_workspace_.arguments.beta = problem_.beta.data();
|
||||
rank_k_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
detail::cublasRankKDispatcher rank_k_op(
|
||||
rank_k_desc,
|
||||
rank_k_workspace_.configuration,
|
||||
rank_k_workspace_.arguments
|
||||
);
|
||||
|
||||
if (rank_k_op.status != Status::kSuccess) {
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kNotRun;
|
||||
return true;
|
||||
}
|
||||
|
||||
results_.back().status = Status::kSuccess;
|
||||
|
||||
status = rank_k_op(handle);
|
||||
|
||||
// Handle errors
|
||||
if (status != CUBLAS_STATUS_SUCCESS) {
|
||||
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// Verify results
|
||||
//
|
||||
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = compare_tensors(
|
||||
options,
|
||||
*rank_k_workspace_.Computed,
|
||||
*rank_k_workspace_.Reference
|
||||
);
|
||||
|
||||
// Save workspace if incorrect
|
||||
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] == Disposition::kIncorrect) {
|
||||
|
||||
save_workspace(
|
||||
device_context,
|
||||
options,
|
||||
rank_k_desc,
|
||||
library::Provider::kCUTLASS,
|
||||
library::Provider::kCUBLAS);
|
||||
}
|
||||
}
|
||||
catch (...) {
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
// Return true means continue profiling
|
||||
return true;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Measures performance results
|
||||
bool Rank2KOperationProfiler::profile(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
|
||||
|
||||
// Initialize structure containing RankK arguments
|
||||
rank_k_workspace_.arguments.A = rank_k_workspace_.A->data();
|
||||
rank_k_workspace_.arguments.B = rank_k_workspace_.B->data();
|
||||
rank_k_workspace_.arguments.C = rank_k_workspace_.C->data();
|
||||
rank_k_workspace_.arguments.D = rank_k_workspace_.Computed->data();
|
||||
rank_k_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
rank_k_workspace_.arguments.beta = problem_.beta.data();
|
||||
rank_k_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
results_.back().status = profile_cutlass_(
|
||||
results_.back().runtime,
|
||||
options,
|
||||
operation,
|
||||
&rank_k_workspace_.arguments,
|
||||
rank_k_workspace_.host_workspace.data(),
|
||||
rank_k_workspace_.device_workspace.data()
|
||||
);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,229 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Defines a math function
|
||||
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <algorithm>
|
||||
#include <unordered_map>
|
||||
|
||||
// CUTLASS Library includes
|
||||
#include "cutlass/blas3.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/util.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
// Profiler includes
|
||||
#include "options.h"
|
||||
#include "device_context.h"
|
||||
#include "operation_profiler.h"
|
||||
#include "performance_result.h"
|
||||
#include "problem_space.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
/// Abstract base class for each math function
|
||||
class Rank2KOperationProfiler : public OperationProfiler {
|
||||
public:
|
||||
|
||||
/// Problem structure obtained from problem space
|
||||
struct RankKProblem {
|
||||
int64_t n;
|
||||
int64_t k;
|
||||
int64_t lda;
|
||||
int64_t ldb;
|
||||
int64_t ldc;
|
||||
FillMode fill_mode;
|
||||
BlasMode blas_mode;
|
||||
std::vector<uint8_t> alpha;
|
||||
std::vector<uint8_t> beta;
|
||||
int64_t split_k_slices;
|
||||
int64_t batch_count;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
RankKProblem():
|
||||
n(16), k(16), lda(0), ldc(0),
|
||||
fill_mode(FillMode::kInvalid), blas_mode(BlasMode::kInvalid),
|
||||
split_k_slices(1), batch_count(1) { }
|
||||
|
||||
/// Parses the problem
|
||||
Status parse(
|
||||
library::RankKDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Total number of bytes loaded
|
||||
int64_t bytes(library::RankKDescription const &operation_desc) const;
|
||||
|
||||
/// Total number of flops computed
|
||||
int64_t flops(library::RankKDescription const &operation_desc) const;
|
||||
|
||||
/// Initializes a performance result
|
||||
void initialize_result(
|
||||
PerformanceResult &result,
|
||||
library::RankKDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space);
|
||||
};
|
||||
|
||||
/// Workspace used
|
||||
struct RankKWorkspace {
|
||||
|
||||
DeviceAllocation *A;
|
||||
DeviceAllocation *B;
|
||||
DeviceAllocation *C;
|
||||
DeviceAllocation *Computed;
|
||||
DeviceAllocation *Reference;
|
||||
|
||||
library::RankKConfiguration configuration;
|
||||
library::RankKArguments arguments;
|
||||
|
||||
/// Buffer used for the operation's host workspace
|
||||
std::vector<uint8_t> host_workspace;
|
||||
|
||||
/// Buffer used for the operations' device workspace
|
||||
DeviceAllocation device_workspace;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
RankKWorkspace():
|
||||
A(nullptr), B(nullptr), C(nullptr), Computed(nullptr), Reference(nullptr) { }
|
||||
};
|
||||
|
||||
protected:
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// GEMM problem obtained from problem space
|
||||
RankKProblem problem_;
|
||||
|
||||
/// Device memory allocations
|
||||
RankKWorkspace rank_k_workspace_;
|
||||
|
||||
|
||||
public:
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Ctor
|
||||
Rank2KOperationProfiler(Options const &options);
|
||||
|
||||
/// Destructor
|
||||
virtual ~Rank2KOperationProfiler();
|
||||
|
||||
/// Prints usage statement for the math function
|
||||
virtual void print_usage(std::ostream &out) const;
|
||||
|
||||
/// Prints examples
|
||||
virtual void print_examples(std::ostream &out) const;
|
||||
|
||||
/// Extracts the problem dimensions
|
||||
virtual Status initialize_configuration(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Initializes workspace
|
||||
virtual Status initialize_workspace(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
virtual bool verify_cutlass(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Measures performance results
|
||||
virtual bool profile(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
protected:
|
||||
|
||||
/// Initializes the performance result
|
||||
void initialize_result_(
|
||||
PerformanceResult &result,
|
||||
Options const &options,
|
||||
library::RankKDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space);
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
bool verify_with_cublas_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,715 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Execution environment
|
||||
|
||||
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
#include <iomanip>
|
||||
#include <ios>
|
||||
|
||||
#include "cutlass/core_io.h"
|
||||
|
||||
#include "cublas_helpers.h"
|
||||
#include "rank_k_operation_profiler.h"
|
||||
#include "gpu_timer.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Ctor
|
||||
RankKOperationProfiler::RankKOperationProfiler(Options const &options):
|
||||
OperationProfiler(
|
||||
options,
|
||||
library::OperationKind::kRankK,
|
||||
{
|
||||
{ArgumentTypeID::kEnumerated, {"rank_k_kind"}, "Variant of RankK (universal)"},
|
||||
{ArgumentTypeID::kInteger, {"n", "problem-size::n"}, "N dimension of the RankK problem space"},
|
||||
{ArgumentTypeID::kInteger, {"k", "problem-size::k"}, "K dimension of the RankK problem space"},
|
||||
{ArgumentTypeID::kTensor, {"A"}, "Tensor storing the A operand"},
|
||||
{ArgumentTypeID::kTensor, {"C"}, "Tensor storing the C operand"},
|
||||
{ArgumentTypeID::kEnumerated, {"fill_mode"}, "Fill Mode for RankK kernel (lower or upper)"},
|
||||
{ArgumentTypeID::kEnumerated, {"blas_mode"}, "Blas Mode for RankK kernel (symmetric or hermitian)"},
|
||||
{ArgumentTypeID::kScalar, {"alpha", "epilogue::alpha"}, "Epilogue scalar alpha"},
|
||||
{ArgumentTypeID::kScalar, {"beta", "epilogue::beta"}, "Epilogue scalar beta"},
|
||||
{ArgumentTypeID::kInteger, {"split_k_slices", "split-k-slices"}, "Number of partitions of K dimension"},
|
||||
{ArgumentTypeID::kInteger, {"batch_count", "batch-count"}, "Number of RankK computed in one batch"},
|
||||
},
|
||||
{ library::Provider::kCUBLAS}
|
||||
) {
|
||||
description_ = " Rank-k Update. D = alpha * A*A^T + beta * C (symmetric) or D = alpha * A*A^H + beta * C (hermitian)";
|
||||
}
|
||||
|
||||
/// Destructor
|
||||
RankKOperationProfiler::~RankKOperationProfiler() {
|
||||
|
||||
}
|
||||
|
||||
/// Prints usage statement for the math function
|
||||
void RankKOperationProfiler::print_usage(std::ostream &out) const {
|
||||
out << "RankK" << "\n\n";
|
||||
|
||||
OperationProfiler::print_usage(out);
|
||||
}
|
||||
|
||||
/// Prints examples
|
||||
void RankKOperationProfiler::print_examples(std::ostream &out) const {
|
||||
|
||||
out << "\nExamples:\n\n"
|
||||
<< "Profile a particular problem size Syrk kernel:\n"
|
||||
<< " $ cutlass_profiler --operation=rank_k --blas_mode=symmetric --n=1024 --k=128\n\n"
|
||||
|
||||
<< "Profile a particular problem size Herk kernel:\n"
|
||||
<< " $ cutlass_profiler --operation=rank_k --blas_mode=hermitian --n=1024 --k=128\n\n"
|
||||
|
||||
<< "Schmoo over problem size and beta:\n"
|
||||
<< " $ cutlass_profiler --operation=rank_k --n=1024:4096:256 --k=128:8192:128 --beta=0,1,2.5\n\n"
|
||||
|
||||
<< "Schmoo over accumulator types:\n"
|
||||
<< " $ cutlass_profiler --operation=rank_k --accumulator-type=f16,f32\n\n"
|
||||
|
||||
<< "Schmoo over fill modees:\n"
|
||||
<< " $ cutlass_profiler --operation=rank_k --fill_mode=lower/upper\n\n"
|
||||
|
||||
<< "Run when A is f16 with column-major or A is any datatype with row-major (For column major, use column, col, or n. For row major use, row or t):\n"
|
||||
<< " $ cutlass_profiler --operation=rank_k --A=f16:column or --A=*:row\n\n"
|
||||
|
||||
<< "Using various input value distribution:\n"
|
||||
<< " $ cutlass_profiler --operation=rank_k --dist=uniform,min:0,max:3\n"
|
||||
<< " $ cutlass_profiler --operation=rank_k --dist=gaussian,mean:0,stddev:3\n"
|
||||
<< " $ cutlass_profiler --operation=rank_k --dist=sequential,start:0,delta:1\n\n"
|
||||
|
||||
<< "Run a kernel with cta tile size of 256x128x32 and save workspace if results are incorrect (note that --cta-tile::k=32 is default cta-tile size):\n"
|
||||
<< " $ cutlass_profiler --operation=rank_k --cta_m=256 --cta_n=128 --cta_k=32 --save-workspace=incorrect\n\n"
|
||||
|
||||
<< "Test your changes to rank_k kernels with a quick functional test and save results in functional-test.csv:\n"
|
||||
<< " $ cutlass_profiler --operation=rank_k \\ \n"
|
||||
<< " --n=8,56,120,136,256,264,512,520,1024,1032,4096,8192,16384 \\ \n"
|
||||
<< " --k=8,16,32,64,128,256,288,384,504,512,520 \\ \n"
|
||||
<< " --beta=0,1,2 --profiling-iterations=1 \\ \n"
|
||||
<< " --providers=cutlass --output=functional-test.csv\n\n";
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if 0
|
||||
// used this for debugging
|
||||
static std::string byte_string(std::vector<uint8_t> const &bytes) {
|
||||
std::stringstream ss;
|
||||
|
||||
ss << "0x";
|
||||
|
||||
for (size_t idx = bytes.size(); idx > 0; --idx) {
|
||||
ss << std::hex << std::setw(2) << std::setfill('0') << uint32_t(bytes.at(idx - 1));
|
||||
}
|
||||
|
||||
return ss.str();
|
||||
}
|
||||
#endif
|
||||
|
||||
Status RankKOperationProfiler::RankKProblem::parse(
|
||||
library::RankKDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
if (!arg_as_int(this->n, "n", problem_space, problem)) {
|
||||
// default value
|
||||
this->n = 1024;
|
||||
}
|
||||
|
||||
if (!arg_as_int(this->k, "k", problem_space, problem)) {
|
||||
// default value
|
||||
this->k = 1024;
|
||||
}
|
||||
|
||||
if (!arg_as_int(this->split_k_slices, "split_k_slices", problem_space, problem)) {
|
||||
// default value
|
||||
this->split_k_slices = 1;
|
||||
}
|
||||
|
||||
if (!arg_as_int(this->batch_count, "batch_count", problem_space, problem)) {
|
||||
// default value
|
||||
this->batch_count = 1;
|
||||
}
|
||||
|
||||
if (this->split_k_slices > 1 && this->batch_count > 1) {
|
||||
// At least one of these must be one
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!tensor_description_satisfies(operation_desc.A, "A", problem_space, problem)) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!tensor_description_satisfies(operation_desc.C, "C", problem_space, problem)) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!arg_as_scalar(
|
||||
this->alpha,
|
||||
operation_desc.element_epilogue,
|
||||
"alpha",
|
||||
problem_space,
|
||||
problem)) {
|
||||
|
||||
if (!cast_from_double(this->alpha, operation_desc.element_epilogue, 1)) {
|
||||
return Status::kErrorInternal;
|
||||
}
|
||||
}
|
||||
|
||||
if (!arg_as_scalar(
|
||||
this->beta,
|
||||
operation_desc.element_epilogue,
|
||||
"beta",
|
||||
problem_space,
|
||||
problem)) {
|
||||
|
||||
if (!cast_from_double(this->beta, operation_desc.element_epilogue, 0)) {
|
||||
return Status::kErrorInternal;
|
||||
}
|
||||
}
|
||||
|
||||
this->lda = DeviceAllocation::get_packed_layout(
|
||||
operation_desc.A.layout, {int(this->n), int(this->k)}).front();
|
||||
|
||||
this->ldc = DeviceAllocation::get_packed_layout(
|
||||
operation_desc.C.layout, {int(this->n), int(this->n)}).front();
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/// Total number of bytes loaded
|
||||
int64_t RankKOperationProfiler::RankKProblem::bytes(library::RankKDescription const &operation_desc) const {
|
||||
// Input bytes read and Output bytes written for the gemm problem
|
||||
int64_t bytes =
|
||||
int64_t(library::sizeof_bits(operation_desc.A.element) * n / 8) * k +
|
||||
int64_t(library::sizeof_bits(operation_desc.A.element) * n / 8) * k +
|
||||
// Half matrix including the diagonal will have (N*(N+1))/2 elements
|
||||
int64_t(library::sizeof_bits(operation_desc.C.element) * n / 8) * (n+1) / 2;
|
||||
|
||||
// Set is_beta_zero true if beta is zero
|
||||
bool is_beta_zero = std::all_of(beta.begin(), beta.end(), [](uint8_t i) { return i==0; });
|
||||
|
||||
// Output bytes read for the gemm problem for non-zero beta values
|
||||
if (!is_beta_zero) {
|
||||
bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * n / 8) * (n+1) / 2;
|
||||
}
|
||||
|
||||
bytes *= batch_count;
|
||||
|
||||
return bytes;
|
||||
}
|
||||
|
||||
/// Total number of flops computed
|
||||
int64_t RankKOperationProfiler::RankKProblem::flops(library::RankKDescription const &operation_desc) const {
|
||||
|
||||
// FLOPs = 2 * n(n+1)k/2 [mma] + 2 * n(n+1)/2 [epilogue]
|
||||
// FLOPs = n(n+1)(k + 1)
|
||||
int64_t flops_ = n * (n + 1) * (k + 1);
|
||||
|
||||
// complex-valued support
|
||||
switch (operation_desc.tile_description.math_instruction.math_operation) {
|
||||
case library::MathOperationID::kMultiplyAddComplex:
|
||||
flops_ *= 4;
|
||||
break;
|
||||
|
||||
case library::MathOperationID::kMultiplyAddComplexFastF32:
|
||||
flops_ *= 4;
|
||||
break;
|
||||
|
||||
case library::MathOperationID::kMultiplyAddGaussianComplex:
|
||||
flops_ *= 3;
|
||||
break;
|
||||
|
||||
default: break;
|
||||
}
|
||||
|
||||
return flops_;
|
||||
}
|
||||
|
||||
/// Initializes a performance result
|
||||
void RankKOperationProfiler::RankKProblem::initialize_result(
|
||||
PerformanceResult &result,
|
||||
library::RankKDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space) {
|
||||
|
||||
result.arguments.resize(problem_space.rank());
|
||||
|
||||
set_argument(result, "rank_k_kind", problem_space, library::to_string(operation_desc.rank_k_kind));
|
||||
|
||||
set_argument(result, "A", problem_space,
|
||||
std::string(library::to_string(operation_desc.A.element)) + ":" + library::to_string(operation_desc.A.layout));
|
||||
|
||||
set_argument(result, "C", problem_space,
|
||||
std::string(library::to_string(operation_desc.C.element)) + ":" + library::to_string(operation_desc.C.layout));
|
||||
|
||||
set_argument(result, "fill_mode", problem_space, library::to_string(operation_desc.fill_mode));
|
||||
|
||||
set_argument(result, "blas_mode", problem_space, library::to_string(operation_desc.blas_mode));
|
||||
|
||||
set_argument(result, "n", problem_space, n);
|
||||
set_argument(result, "k", problem_space, k);
|
||||
|
||||
set_argument(result, "split_k_slices", problem_space, split_k_slices);
|
||||
set_argument(result, "batch_count", problem_space, batch_count);
|
||||
|
||||
set_argument(result, "alpha", problem_space,
|
||||
library::lexical_cast(alpha, operation_desc.element_epilogue));
|
||||
|
||||
set_argument(result, "beta", problem_space,
|
||||
library::lexical_cast(beta, operation_desc.element_epilogue));
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Extracts the problem dimensions
|
||||
Status RankKOperationProfiler::initialize_configuration(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
library::RankKDescription const &operation_desc =
|
||||
static_cast<library::RankKDescription const &>(operation->description());
|
||||
|
||||
if (operation_desc.rank_k_kind != library::RankKKind::kUniversal) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
Status status = problem_.parse(operation_desc, problem_space, problem);
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
rank_k_workspace_.configuration.problem_size.m() = int(problem_.n);
|
||||
rank_k_workspace_.configuration.problem_size.n() = int(problem_.n);
|
||||
rank_k_workspace_.configuration.problem_size.k() = int(problem_.k);
|
||||
rank_k_workspace_.configuration.lda = problem_.lda;
|
||||
rank_k_workspace_.configuration.ldc = problem_.ldc;
|
||||
rank_k_workspace_.configuration.ldd = problem_.ldc;
|
||||
//rank_k_workspace_.configuration.split_k_slices = int(problem_.split_k_slices);
|
||||
rank_k_workspace_.configuration.batch_count = int(problem_.split_k_slices);
|
||||
|
||||
rank_k_workspace_.arguments.A = nullptr;
|
||||
rank_k_workspace_.arguments.C = nullptr;
|
||||
rank_k_workspace_.arguments.D = nullptr;
|
||||
rank_k_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
rank_k_workspace_.arguments.beta = problem_.beta.data();
|
||||
rank_k_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
initialize_result_(this->model_result_, options, operation_desc, problem_space);
|
||||
|
||||
return operation->can_implement(&rank_k_workspace_.configuration, &rank_k_workspace_.arguments);
|
||||
}
|
||||
|
||||
/// Initializes the performance result
|
||||
void RankKOperationProfiler::initialize_result_(
|
||||
PerformanceResult &result,
|
||||
Options const &options,
|
||||
library::RankKDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space) {
|
||||
|
||||
result.provider = library::Provider::kCUTLASS;
|
||||
result.disposition = Disposition::kNotRun;
|
||||
result.status = Status::kSuccess;
|
||||
result.operation_name = operation_desc.name;
|
||||
|
||||
problem_.initialize_result(result, operation_desc, problem_space);
|
||||
|
||||
OperationProfiler::initialize_result_(result, operation_desc, problem_space);
|
||||
|
||||
|
||||
result.bytes = problem_.bytes(operation_desc);
|
||||
result.flops = problem_.flops(operation_desc);
|
||||
|
||||
result.runtime = 0;
|
||||
|
||||
// complex-valued support
|
||||
switch (operation_desc.tile_description.math_instruction.math_operation) {
|
||||
case library::MathOperationID::kMultiplyAddComplex:
|
||||
result.flops *= 4;
|
||||
break;
|
||||
|
||||
case library::MathOperationID::kMultiplyAddComplexFastF32:
|
||||
result.flops *= 4;
|
||||
break;
|
||||
|
||||
default: break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/// Initializes workspace
|
||||
Status RankKOperationProfiler::initialize_workspace(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
library::RankKDescription const &operation_desc =
|
||||
static_cast<library::RankKDescription const &>(operation->description());
|
||||
|
||||
if (options.execution_mode != ExecutionMode::kDryRun) {
|
||||
|
||||
rank_k_workspace_.A = device_context.allocate_tensor(
|
||||
options,
|
||||
"A",
|
||||
operation_desc.A.element,
|
||||
operation_desc.A.layout,
|
||||
{int(problem_.n), int(problem_.k)},
|
||||
{int(problem_.lda)}
|
||||
);
|
||||
|
||||
rank_k_workspace_.C = device_context.allocate_tensor(
|
||||
options,
|
||||
"C",
|
||||
operation_desc.C.element,
|
||||
operation_desc.C.layout,
|
||||
{int(problem_.n), int(problem_.n)},
|
||||
{int(problem_.ldc)},
|
||||
1 // batch_count = 1, default
|
||||
);
|
||||
|
||||
rank_k_workspace_.Computed = device_context.allocate_tensor(
|
||||
"D",
|
||||
operation_desc.C.element,
|
||||
operation_desc.C.layout,
|
||||
{int(problem_.n), int(problem_.n)},
|
||||
{int(problem_.ldc)}
|
||||
);
|
||||
|
||||
rank_k_workspace_.Reference = device_context.allocate_tensor(
|
||||
"Reference",
|
||||
operation_desc.C.element,
|
||||
operation_desc.C.layout,
|
||||
{int(problem_.n), int(problem_.n)},
|
||||
{int(problem_.ldc)}
|
||||
);
|
||||
|
||||
rank_k_workspace_.Computed->copy_from_device(rank_k_workspace_.C->data());
|
||||
rank_k_workspace_.Reference->copy_from_device(rank_k_workspace_.C->data());
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// Initialize the CUTLASS operation
|
||||
//
|
||||
Status status = Status::kSuccess;
|
||||
|
||||
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
|
||||
|
||||
if (options.execution_mode != ExecutionMode::kDryRun) {
|
||||
|
||||
uint64_t workspace_size = operation->get_host_workspace_size(&rank_k_workspace_.configuration);
|
||||
rank_k_workspace_.host_workspace.resize(workspace_size, 0);
|
||||
|
||||
workspace_size = operation->get_device_workspace_size(&rank_k_workspace_.configuration);
|
||||
rank_k_workspace_.device_workspace.reset(library::NumericTypeID::kU8, workspace_size);
|
||||
|
||||
status = operation->initialize(
|
||||
&rank_k_workspace_.configuration,
|
||||
rank_k_workspace_.host_workspace.data(),
|
||||
rank_k_workspace_.device_workspace.data());
|
||||
}
|
||||
|
||||
//
|
||||
// If CUTLASS is enabled, generate a result for it
|
||||
//
|
||||
results_.push_back(model_result_);
|
||||
results_.back().provider = library::Provider::kCUTLASS;
|
||||
results_.back().op_kind = library::OperationKind::kRankK;
|
||||
results_.back().disposition = Disposition::kNotRun;
|
||||
|
||||
for(auto provider : verification_providers_) {
|
||||
results_.back().verification_map[provider] = Disposition::kNotRun;
|
||||
}
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
bool RankKOperationProfiler::verify_cutlass(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
if (!options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (options.execution_mode == ExecutionMode::kDryRun) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Initialize structure containing RankK arguments
|
||||
rank_k_workspace_.arguments.A = rank_k_workspace_.A->data();
|
||||
rank_k_workspace_.arguments.C = rank_k_workspace_.C->data();
|
||||
rank_k_workspace_.arguments.D = rank_k_workspace_.Computed->data();
|
||||
rank_k_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
rank_k_workspace_.arguments.beta = problem_.beta.data();
|
||||
rank_k_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
//
|
||||
// Run the CUTLASS operation
|
||||
//
|
||||
|
||||
results_.back().status = operation->run(
|
||||
&rank_k_workspace_.arguments,
|
||||
rank_k_workspace_.host_workspace.data(),
|
||||
rank_k_workspace_.device_workspace.data());
|
||||
|
||||
if (results_.back().status != Status::kSuccess) {
|
||||
results_.back().disposition = Disposition::kFailed;
|
||||
return false;
|
||||
}
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
if (result != cudaSuccess) {
|
||||
results_.back().disposition = Disposition::kFailed;
|
||||
return false;
|
||||
}
|
||||
|
||||
// CUTLASS op ran the but not yet verified against any verification provider
|
||||
results_.back().disposition = Disposition::kNotVerified;
|
||||
|
||||
//
|
||||
// Run verification providers
|
||||
//
|
||||
|
||||
if (options.verification.enabled) {
|
||||
|
||||
#if CUTLASS_ENABLE_CUBLAS
|
||||
if (options.verification.provider_enabled(library::Provider::kCUBLAS)) {
|
||||
|
||||
// Guard against unsupported cases
|
||||
auto const & rank_k_desc = static_cast<library::RankKDescription const &>(operation->description());
|
||||
|
||||
if (cublas_satisfies(rank_k_desc) == Status::kSuccess) {
|
||||
|
||||
// call cublas verification if supported
|
||||
verify_with_cublas_(
|
||||
options,
|
||||
report,
|
||||
device_context,
|
||||
operation,
|
||||
problem_space,
|
||||
problem);
|
||||
}
|
||||
|
||||
else {
|
||||
// set verification map for cublas to not supported
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kNotSupported;
|
||||
}
|
||||
}
|
||||
#endif // #if CUTLASS_ENABLE_CUBLAS
|
||||
|
||||
// Update disposition to worst case verification outcome among all
|
||||
// verification providers which are supported
|
||||
bool is_any_verification_run_passed = false;
|
||||
for(auto &m : results_.back().verification_map) {
|
||||
if(m.second == Disposition::kFailed || m.second == Disposition::kIncorrect) {
|
||||
results_.back().disposition = m.second;
|
||||
return true;
|
||||
}
|
||||
if(!is_any_verification_run_passed && m.second == Disposition::kPassed) {
|
||||
is_any_verification_run_passed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if(is_any_verification_run_passed) {
|
||||
results_.back().disposition = Disposition::kPassed;
|
||||
}
|
||||
}
|
||||
|
||||
// Return true means continue profiling
|
||||
return true;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
bool RankKOperationProfiler::verify_with_cublas_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
|
||||
#if CUTLASS_ENABLE_CUBLAS
|
||||
|
||||
library::RankKDescription const &rank_k_desc =
|
||||
static_cast<library::RankKDescription const &>(operation->description());
|
||||
|
||||
//
|
||||
// Construct cuBLAS operators
|
||||
//
|
||||
|
||||
CublasCreate handle;
|
||||
cublasStatus_t status = handle.get_cublas_create_status();
|
||||
|
||||
if (status != CUBLAS_STATUS_SUCCESS) {
|
||||
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// Initialize state
|
||||
//
|
||||
|
||||
try {
|
||||
|
||||
//
|
||||
// Construct dispatcher to cublas<t>Syrk()
|
||||
//
|
||||
|
||||
// Initialize structure containing RankK arguments
|
||||
rank_k_workspace_.arguments.A = rank_k_workspace_.A->data();
|
||||
rank_k_workspace_.arguments.C = rank_k_workspace_.Reference->data();
|
||||
rank_k_workspace_.arguments.D = rank_k_workspace_.Reference->data();
|
||||
rank_k_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
rank_k_workspace_.arguments.beta = problem_.beta.data();
|
||||
rank_k_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
detail::cublasRankKDispatcher rank_k_op(
|
||||
rank_k_desc,
|
||||
rank_k_workspace_.configuration,
|
||||
rank_k_workspace_.arguments
|
||||
);
|
||||
|
||||
if (rank_k_op.status != Status::kSuccess) {
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kNotRun;
|
||||
return true;
|
||||
}
|
||||
|
||||
results_.back().status = Status::kSuccess;
|
||||
|
||||
status = rank_k_op(handle);
|
||||
|
||||
// Handle errors
|
||||
if (status != CUBLAS_STATUS_SUCCESS) {
|
||||
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// Verify results
|
||||
//
|
||||
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = compare_tensors(
|
||||
options,
|
||||
*rank_k_workspace_.Computed,
|
||||
*rank_k_workspace_.Reference
|
||||
);
|
||||
|
||||
// Save workspace if incorrect
|
||||
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] == Disposition::kIncorrect) {
|
||||
|
||||
save_workspace(
|
||||
device_context,
|
||||
options,
|
||||
rank_k_desc,
|
||||
library::Provider::kCUTLASS,
|
||||
library::Provider::kCUBLAS);
|
||||
}
|
||||
}
|
||||
catch (...) {
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
// Return true means continue profiling
|
||||
return true;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Measures performance results
|
||||
bool RankKOperationProfiler::profile(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
|
||||
|
||||
// Initialize structure containing RankK arguments
|
||||
rank_k_workspace_.arguments.A = rank_k_workspace_.A->data();
|
||||
rank_k_workspace_.arguments.C = rank_k_workspace_.C->data();
|
||||
rank_k_workspace_.arguments.D = rank_k_workspace_.Computed->data();
|
||||
rank_k_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
rank_k_workspace_.arguments.beta = problem_.beta.data();
|
||||
rank_k_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
results_.back().status = profile_cutlass_(
|
||||
results_.back().runtime,
|
||||
options,
|
||||
operation,
|
||||
&rank_k_workspace_.arguments,
|
||||
rank_k_workspace_.host_workspace.data(),
|
||||
rank_k_workspace_.device_workspace.data()
|
||||
);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,227 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Defines a math function
|
||||
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <algorithm>
|
||||
#include <unordered_map>
|
||||
|
||||
// CUTLASS Library includes
|
||||
#include "cutlass/blas3.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/util.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
// Profiler includes
|
||||
#include "options.h"
|
||||
#include "device_context.h"
|
||||
#include "operation_profiler.h"
|
||||
#include "performance_result.h"
|
||||
#include "problem_space.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
/// Abstract base class for each math function
|
||||
class RankKOperationProfiler : public OperationProfiler {
|
||||
public:
|
||||
|
||||
/// Problem structure obtained from problem space
|
||||
struct RankKProblem {
|
||||
int64_t n;
|
||||
int64_t k;
|
||||
int64_t lda;
|
||||
int64_t ldc;
|
||||
FillMode fill_mode;
|
||||
BlasMode blas_mode;
|
||||
std::vector<uint8_t> alpha;
|
||||
std::vector<uint8_t> beta;
|
||||
int64_t split_k_slices;
|
||||
int64_t batch_count;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
RankKProblem():
|
||||
n(16), k(16), lda(0), ldc(0),
|
||||
fill_mode(FillMode::kInvalid), blas_mode(BlasMode::kInvalid),
|
||||
split_k_slices(1), batch_count(1) { }
|
||||
|
||||
/// Parses the problem
|
||||
Status parse(
|
||||
library::RankKDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Total number of bytes loaded
|
||||
int64_t bytes(library::RankKDescription const &operation_desc) const;
|
||||
|
||||
/// Total number of flops computed
|
||||
int64_t flops(library::RankKDescription const &operation_desc) const;
|
||||
|
||||
/// Initializes a performance result
|
||||
void initialize_result(
|
||||
PerformanceResult &result,
|
||||
library::RankKDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space);
|
||||
};
|
||||
|
||||
/// Workspace used
|
||||
struct RankKWorkspace {
|
||||
|
||||
DeviceAllocation *A;
|
||||
DeviceAllocation *C;
|
||||
DeviceAllocation *Computed;
|
||||
DeviceAllocation *Reference;
|
||||
|
||||
library::RankKConfiguration configuration;
|
||||
library::RankKArguments arguments;
|
||||
|
||||
/// Buffer used for the operation's host workspace
|
||||
std::vector<uint8_t> host_workspace;
|
||||
|
||||
/// Buffer used for the operations' device workspace
|
||||
DeviceAllocation device_workspace;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
RankKWorkspace():
|
||||
A(nullptr), C(nullptr), Computed(nullptr), Reference(nullptr) { }
|
||||
};
|
||||
|
||||
protected:
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// GEMM problem obtained from problem space
|
||||
RankKProblem problem_;
|
||||
|
||||
/// Device memory allocations
|
||||
RankKWorkspace rank_k_workspace_;
|
||||
|
||||
|
||||
public:
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Ctor
|
||||
RankKOperationProfiler(Options const &options);
|
||||
|
||||
/// Destructor
|
||||
virtual ~RankKOperationProfiler();
|
||||
|
||||
/// Prints usage statement for the math function
|
||||
virtual void print_usage(std::ostream &out) const;
|
||||
|
||||
/// Prints examples
|
||||
virtual void print_examples(std::ostream &out) const;
|
||||
|
||||
/// Extracts the problem dimensions
|
||||
virtual Status initialize_configuration(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Initializes workspace
|
||||
virtual Status initialize_workspace(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
virtual bool verify_cutlass(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Measures performance results
|
||||
virtual bool profile(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
protected:
|
||||
|
||||
/// Initializes the performance result
|
||||
void initialize_result_(
|
||||
PerformanceResult &result,
|
||||
Options const &options,
|
||||
library::RankKDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space);
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
bool verify_with_cublas_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
@@ -0,0 +1,764 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Execution environment
|
||||
|
||||
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
#include <iomanip>
|
||||
#include <ios>
|
||||
|
||||
#include "cutlass/core_io.h"
|
||||
|
||||
#include "cublas_helpers.h"
|
||||
#include "symm_operation_profiler.h"
|
||||
#include "gpu_timer.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Ctor
|
||||
SymmOperationProfiler::SymmOperationProfiler(Options const &options):
|
||||
OperationProfiler(
|
||||
options,
|
||||
library::OperationKind::kSymm,
|
||||
{
|
||||
{ArgumentTypeID::kEnumerated, {"symm_kind"}, "Variant of Symm (universal)"},
|
||||
{ArgumentTypeID::kInteger, {"m", "problem-size::m"}, "M dimension of the Symm problem space"},
|
||||
{ArgumentTypeID::kInteger, {"n", "problem-size::n"}, "N dimension of the Symm problem space"},
|
||||
{ArgumentTypeID::kTensor, {"A"}, "Tensor storing the A operand"},
|
||||
{ArgumentTypeID::kTensor, {"B"}, "Tensor storing the B operand"},
|
||||
{ArgumentTypeID::kTensor, {"C"}, "Tensor storing the C operand"},
|
||||
{ArgumentTypeID::kEnumerated, {"side_mode"}, "Side Mode for Symm kernel (left or right)"},
|
||||
{ArgumentTypeID::kEnumerated, {"fill_mode"}, "Fill Mode for Symm kernel (lower or upper)"},
|
||||
{ArgumentTypeID::kEnumerated, {"blas_mode"}, "Blas Mode for Symm kernel (symmetric or hermitian)"},
|
||||
{ArgumentTypeID::kScalar, {"alpha", "epilogue::alpha"}, "Epilogue scalar alpha"},
|
||||
{ArgumentTypeID::kScalar, {"beta", "epilogue::beta"}, "Epilogue scalar beta"},
|
||||
{ArgumentTypeID::kInteger, {"split_k_slices", "split-k-slices"}, "Number of partitions of K dimension"},
|
||||
{ArgumentTypeID::kInteger, {"batch_count", "batch-count"}, "Number of Symm computed in one batch"},
|
||||
},
|
||||
{ library::Provider::kCUBLAS }
|
||||
) {
|
||||
description_ = " Symmetric Matrix-Matrix Multiplication. D = alpha * A * B OR alpha * B * A + beta * C (where A is symmetric/hermitian)";
|
||||
}
|
||||
|
||||
/// Destructor
|
||||
SymmOperationProfiler::~SymmOperationProfiler() {
|
||||
|
||||
}
|
||||
|
||||
/// Prints usage statement for the math function
|
||||
void SymmOperationProfiler::print_usage(std::ostream &out) const {
|
||||
out << "Symm" << "\n\n";
|
||||
|
||||
OperationProfiler::print_usage(out);
|
||||
}
|
||||
|
||||
/// Prints examples
|
||||
void SymmOperationProfiler::print_examples(std::ostream &out) const {
|
||||
|
||||
out << "\nExamples:\n\n"
|
||||
<< "Profile a particular problem size SYMM kernel:\n"
|
||||
<< " $ cutlass_profiler --operation=Symm --blas_mode=symmetric --m=1024 --n=128\n\n"
|
||||
|
||||
<< "Profile a particular problem size HEMM kernel:\n"
|
||||
<< " $ cutlass_profiler --operation=Symm --blas_mode=hermitian --m=1024 --n=128\n\n"
|
||||
|
||||
<< "Schmoo over problem size and beta:\n"
|
||||
<< " $ cutlass_profiler --operation=Symm --m=1024:4096:256 --n=128:8192:128 --beta=0,1,2.5\n\n"
|
||||
|
||||
<< "Schmoo over accumulator types:\n"
|
||||
<< " $ cutlass_profiler --operation=Symm --accumulator-type=f16,f32\n\n"
|
||||
|
||||
<< "Schmoo over side modees:\n"
|
||||
<< " $ cutlass_profiler --operation=Symm --side_mode=left/right\n\n"
|
||||
|
||||
<< "Schmoo over fill modees:\n"
|
||||
<< " $ cutlass_profiler --operation=Symm --fill_mode=lower/upper\n\n"
|
||||
|
||||
<< "Run when A is f16 with column-major or A is any datatype with row-major (For column major, use column, col, or n. For row major use, row or t):\n"
|
||||
<< " $ cutlass_profiler --operation=Symm --A=f16:column or --A=*:row\n\n"
|
||||
|
||||
<< "Using various input value distribution:\n"
|
||||
<< " $ cutlass_profiler --operation=Symm --dist=uniform,min:0,max:3\n"
|
||||
<< " $ cutlass_profiler --operation=Symm --dist=gaussian,mean:0,stddev:3\n"
|
||||
<< " $ cutlass_profiler --operation=Symm --dist=sequential,start:0,delta:1\n\n"
|
||||
|
||||
<< "Run a kernel with cta tile size of 256x128x32 and save workspace if results are incorrect (note that --cta-tile::k=32 is default cta-tile size):\n"
|
||||
<< " $ cutlass_profiler --operation=Symm --cta_m=256 --cta_n=128 --cta_k=32 --save-workspace=incorrect\n\n"
|
||||
|
||||
<< "Test your changes to symm kernels with a quick functional test and save results in functional-test.csv:\n"
|
||||
<< " $ cutlass_profiler --operation=Symm \\ \n"
|
||||
<< " --m=8,56,120,136,256,264,512,520,1024,1032,4096,8192,16384 \\ \n"
|
||||
<< " --n=8,16,32,64,128,256,288,384,504,512,520 \\ \n"
|
||||
<< " --beta=0,1,2 --profiling-iterations=1 \\ \n"
|
||||
<< " --providers=cutlass --output=functional-test.csv\n\n";
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if 0
|
||||
// used this for debugging
|
||||
static std::string byte_string(std::vector<uint8_t> const &bytes) {
|
||||
std::stringstream ss;
|
||||
|
||||
ss << "0x";
|
||||
|
||||
for (size_t idx = bytes.size(); idx > 0; --idx) {
|
||||
ss << std::hex << std::setw(2) << std::setfill('0') << uint32_t(bytes.at(idx - 1));
|
||||
}
|
||||
|
||||
return ss.str();
|
||||
}
|
||||
#endif
|
||||
|
||||
Status SymmOperationProfiler::SymmProblem::parse(
|
||||
library::SymmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
if (!arg_as_int(this->m, "m", problem_space, problem)) {
|
||||
// default value
|
||||
this->m = 1024;
|
||||
}
|
||||
|
||||
if (!arg_as_int(this->n, "n", problem_space, problem)) {
|
||||
// default value
|
||||
this->n = 1024;
|
||||
}
|
||||
|
||||
if (!arg_as_int(this->split_k_slices, "split_k_slices", problem_space, problem)) {
|
||||
// default value
|
||||
this->split_k_slices = 1;
|
||||
}
|
||||
|
||||
if (!arg_as_int(this->batch_count, "batch_count", problem_space, problem)) {
|
||||
// default value
|
||||
this->batch_count = 1;
|
||||
}
|
||||
|
||||
if (this->split_k_slices > 1 && this->batch_count > 1) {
|
||||
// At least one of these must be one
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!tensor_description_satisfies(operation_desc.A, "A", problem_space, problem)) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!tensor_description_satisfies(operation_desc.B, "B", problem_space, problem)) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!tensor_description_satisfies(operation_desc.C, "C", problem_space, problem)) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!arg_as_scalar(
|
||||
this->alpha,
|
||||
operation_desc.element_epilogue,
|
||||
"alpha",
|
||||
problem_space,
|
||||
problem)) {
|
||||
|
||||
if (!cast_from_double(this->alpha, operation_desc.element_epilogue, 1)) {
|
||||
return Status::kErrorInternal;
|
||||
}
|
||||
}
|
||||
|
||||
if (!arg_as_scalar(
|
||||
this->beta,
|
||||
operation_desc.element_epilogue,
|
||||
"beta",
|
||||
problem_space,
|
||||
problem)) {
|
||||
|
||||
if (!cast_from_double(this->beta, operation_desc.element_epilogue, 0)) {
|
||||
return Status::kErrorInternal;
|
||||
}
|
||||
}
|
||||
|
||||
if (operation_desc.side_mode == SideMode::kLeft) {
|
||||
this->lda = DeviceAllocation::get_packed_layout(
|
||||
operation_desc.A.layout, {int(this->m), int(this->m)}).front();
|
||||
}
|
||||
else if (operation_desc.side_mode == SideMode::kRight) {
|
||||
this->lda = DeviceAllocation::get_packed_layout(
|
||||
operation_desc.A.layout, {int(this->n), int(this->n)}).front();
|
||||
}
|
||||
|
||||
this->ldb = DeviceAllocation::get_packed_layout(
|
||||
operation_desc.B.layout, {int(this->m), int(this->n)}).front();
|
||||
|
||||
this->ldc = DeviceAllocation::get_packed_layout(
|
||||
operation_desc.C.layout, {int(this->m), int(this->n)}).front();
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/// Total number of bytes loaded
|
||||
int64_t SymmOperationProfiler::SymmProblem::bytes(library::SymmDescription const &operation_desc) const {
|
||||
int64_t bytes;
|
||||
// Input bytes read and Output bytes written for the gemm problem
|
||||
// Half matrix including the diagonal will have (X*(X+1))/2 elements
|
||||
if (operation_desc.side_mode == SideMode::kLeft) {
|
||||
bytes =
|
||||
int64_t(library::sizeof_bits(operation_desc.A.element) * m / 8) * (m + 1) / 2 +
|
||||
int64_t(library::sizeof_bits(operation_desc.B.element) * m / 8) * n +
|
||||
int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
|
||||
} else if (operation_desc.side_mode == SideMode::kRight) {
|
||||
bytes =
|
||||
int64_t(library::sizeof_bits(operation_desc.A.element) * n / 8) * (n + 1) / 2 +
|
||||
int64_t(library::sizeof_bits(operation_desc.B.element) * m / 8) * n +
|
||||
int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
|
||||
}
|
||||
// Set is_beta_zero true if beta is zero
|
||||
bool is_beta_zero = std::all_of(beta.begin(), beta.end(), [](uint8_t i) { return i==0; });
|
||||
|
||||
// Output bytes read for the gemm problem for non-zero beta values
|
||||
if (!is_beta_zero) {
|
||||
bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
|
||||
}
|
||||
|
||||
bytes *= batch_count;
|
||||
|
||||
return bytes;
|
||||
}
|
||||
|
||||
/// Total number of flops computed
|
||||
int64_t SymmOperationProfiler::SymmProblem::flops(library::SymmDescription const &operation_desc) const {
|
||||
|
||||
// FLOPs for first TRMM kernel (with diagonal) = 2 * [ ( M * (M+1)/2 * N ) ] // Beta is zero
|
||||
// FLOPs for second TRMM kernel (with diagonal) = 2 * [ ( M * (M-1)/2 * N ) ] // Beta is zero
|
||||
// FLOPs = m*(m+1)*n [mma1] + m*(m-1)*n [mma2] + 2*m*n [epilogue]
|
||||
// FLOPs = 2*m*n(m+1) for left side mode
|
||||
// FLOPs can also be calculated to be same as GEMM with correct value for 'k' as below.
|
||||
int64_t k = (operation_desc.side_mode == SideMode::kLeft) ? int64_t(m) : int64_t(n);
|
||||
int64_t flops_ = (int64_t(m) * n * k + m * n) * 2;
|
||||
|
||||
// complex-valued support
|
||||
switch (operation_desc.tile_description.math_instruction.math_operation) {
|
||||
case library::MathOperationID::kMultiplyAddComplex:
|
||||
flops_ *= 4;
|
||||
break;
|
||||
|
||||
case library::MathOperationID::kMultiplyAddComplexFastF32:
|
||||
flops_ *= 4;
|
||||
break;
|
||||
|
||||
case library::MathOperationID::kMultiplyAddGaussianComplex:
|
||||
flops_ *= 3;
|
||||
break;
|
||||
|
||||
default: break;
|
||||
}
|
||||
|
||||
return flops_;
|
||||
}
|
||||
|
||||
/// Initializes a performance result
|
||||
void SymmOperationProfiler::SymmProblem::initialize_result(
|
||||
PerformanceResult &result,
|
||||
library::SymmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space) {
|
||||
|
||||
result.arguments.resize(problem_space.rank());
|
||||
|
||||
set_argument(result, "symm_kind", problem_space, library::to_string(operation_desc.symm_kind));
|
||||
|
||||
set_argument(result, "A", problem_space,
|
||||
std::string(library::to_string(operation_desc.A.element)) + ":" + library::to_string(operation_desc.A.layout));
|
||||
|
||||
set_argument(result, "B", problem_space,
|
||||
std::string(library::to_string(operation_desc.B.element)) + ":" + library::to_string(operation_desc.B.layout));
|
||||
|
||||
set_argument(result, "C", problem_space,
|
||||
std::string(library::to_string(operation_desc.C.element)) + ":" + library::to_string(operation_desc.C.layout));
|
||||
|
||||
set_argument(result, "side_mode", problem_space, library::to_string(operation_desc.side_mode));
|
||||
|
||||
set_argument(result, "fill_mode", problem_space, library::to_string(operation_desc.fill_mode));
|
||||
|
||||
set_argument(result, "blas_mode", problem_space, library::to_string(operation_desc.blas_mode));
|
||||
|
||||
set_argument(result, "m", problem_space, m);
|
||||
set_argument(result, "n", problem_space, n);
|
||||
|
||||
set_argument(result, "split_k_slices", problem_space, split_k_slices);
|
||||
set_argument(result, "batch_count", problem_space, batch_count);
|
||||
|
||||
set_argument(result, "alpha", problem_space,
|
||||
library::lexical_cast(alpha, operation_desc.element_epilogue));
|
||||
|
||||
set_argument(result, "beta", problem_space,
|
||||
library::lexical_cast(beta, operation_desc.element_epilogue));
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Extracts the problem dimensions
|
||||
Status SymmOperationProfiler::initialize_configuration(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
library::SymmDescription const &operation_desc =
|
||||
static_cast<library::SymmDescription const &>(operation->description());
|
||||
|
||||
if (operation_desc.symm_kind != library::SymmKind::kUniversal) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
Status status = problem_.parse(operation_desc, problem_space, problem);
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
symm_workspace_.configuration.problem_size.m() = int(problem_.m);
|
||||
symm_workspace_.configuration.problem_size.n() = int(problem_.n);
|
||||
symm_workspace_.configuration.problem_size.k() = (operation_desc.side_mode == SideMode::kLeft)
|
||||
? int(problem_.m) : int(problem_.n);
|
||||
symm_workspace_.configuration.lda = problem_.lda;
|
||||
symm_workspace_.configuration.ldb = problem_.ldb;
|
||||
symm_workspace_.configuration.ldc = problem_.ldc;
|
||||
symm_workspace_.configuration.ldd = problem_.ldc;
|
||||
//symm_workspace_.configuration.split_k_slices = int(problem_.split_k_slices);
|
||||
symm_workspace_.configuration.batch_count = int(problem_.split_k_slices);
|
||||
|
||||
symm_workspace_.arguments.A = nullptr;
|
||||
symm_workspace_.arguments.B = nullptr;
|
||||
symm_workspace_.arguments.C = nullptr;
|
||||
symm_workspace_.arguments.D = nullptr;
|
||||
symm_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
symm_workspace_.arguments.beta = problem_.beta.data();
|
||||
symm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
initialize_result_(this->model_result_, options, operation_desc, problem_space);
|
||||
|
||||
return operation->can_implement(&symm_workspace_.configuration, &symm_workspace_.arguments);
|
||||
}
|
||||
|
||||
/// Initializes the performance result
|
||||
void SymmOperationProfiler::initialize_result_(
|
||||
PerformanceResult &result,
|
||||
Options const &options,
|
||||
library::SymmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space) {
|
||||
|
||||
result.provider = library::Provider::kCUTLASS;
|
||||
result.disposition = Disposition::kNotRun;
|
||||
result.status = Status::kSuccess;
|
||||
result.operation_name = operation_desc.name;
|
||||
|
||||
problem_.initialize_result(result, operation_desc, problem_space);
|
||||
|
||||
OperationProfiler::initialize_result_(result, operation_desc, problem_space);
|
||||
|
||||
|
||||
result.bytes = problem_.bytes(operation_desc);
|
||||
result.flops = problem_.flops(operation_desc);
|
||||
result.runtime = 0;
|
||||
|
||||
|
||||
}
|
||||
|
||||
/// Initializes workspace
|
||||
Status SymmOperationProfiler::initialize_workspace(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
library::SymmDescription const &operation_desc =
|
||||
static_cast<library::SymmDescription const &>(operation->description());
|
||||
|
||||
if (options.execution_mode != ExecutionMode::kDryRun) {
|
||||
|
||||
if (operation_desc.side_mode == SideMode::kLeft) {
|
||||
symm_workspace_.A = device_context.allocate_tensor(
|
||||
options,
|
||||
"A",
|
||||
operation_desc.A.element,
|
||||
operation_desc.A.layout,
|
||||
{int(problem_.m), int(problem_.m)},
|
||||
{int(problem_.lda)},
|
||||
1 // batch_count = 1, default
|
||||
);
|
||||
} else if (operation_desc.side_mode == SideMode::kRight) {
|
||||
symm_workspace_.A = device_context.allocate_tensor(
|
||||
options,
|
||||
"A",
|
||||
operation_desc.A.element,
|
||||
operation_desc.A.layout,
|
||||
{int(problem_.n), int(problem_.n)},
|
||||
{int(problem_.lda)},
|
||||
1 // batch_count = 1, default
|
||||
);
|
||||
}
|
||||
|
||||
symm_workspace_.B = device_context.allocate_tensor(
|
||||
options,
|
||||
"B",
|
||||
operation_desc.B.element,
|
||||
operation_desc.B.layout,
|
||||
{int(problem_.m), int(problem_.n)},
|
||||
{int(problem_.ldb)}
|
||||
);
|
||||
|
||||
symm_workspace_.C = device_context.allocate_tensor(
|
||||
options,
|
||||
"C",
|
||||
operation_desc.C.element,
|
||||
operation_desc.C.layout,
|
||||
{int(problem_.m), int(problem_.n)},
|
||||
{int(problem_.ldc)},
|
||||
1 // batch_count = 1, default
|
||||
);
|
||||
|
||||
symm_workspace_.Computed = device_context.allocate_tensor(
|
||||
"D",
|
||||
operation_desc.C.element,
|
||||
operation_desc.C.layout,
|
||||
{int(problem_.m), int(problem_.n)},
|
||||
{int(problem_.ldc)}
|
||||
);
|
||||
|
||||
symm_workspace_.Reference = device_context.allocate_tensor(
|
||||
"Reference",
|
||||
operation_desc.C.element,
|
||||
operation_desc.C.layout,
|
||||
{int(problem_.m), int(problem_.n)},
|
||||
{int(problem_.ldc)}
|
||||
);
|
||||
|
||||
symm_workspace_.Computed->copy_from_device(symm_workspace_.C->data());
|
||||
symm_workspace_.Reference->copy_from_device(symm_workspace_.C->data());
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// Initialize the CUTLASS operation
|
||||
//
|
||||
Status status = Status::kSuccess;
|
||||
|
||||
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
|
||||
|
||||
if (options.execution_mode != ExecutionMode::kDryRun) {
|
||||
|
||||
uint64_t workspace_size = operation->get_host_workspace_size(&symm_workspace_.configuration);
|
||||
symm_workspace_.host_workspace.resize(workspace_size, 0);
|
||||
|
||||
workspace_size = operation->get_device_workspace_size(&symm_workspace_.configuration);
|
||||
symm_workspace_.device_workspace.reset(library::NumericTypeID::kU8, workspace_size);
|
||||
|
||||
status = operation->initialize(
|
||||
&symm_workspace_.configuration,
|
||||
symm_workspace_.host_workspace.data(),
|
||||
symm_workspace_.device_workspace.data());
|
||||
}
|
||||
|
||||
//
|
||||
// If CUTLASS is enabled, generate a result for it
|
||||
//
|
||||
results_.push_back(model_result_);
|
||||
results_.back().provider = library::Provider::kCUTLASS;
|
||||
results_.back().op_kind = library::OperationKind::kSymm;
|
||||
results_.back().disposition = Disposition::kNotRun;
|
||||
|
||||
for(auto provider : verification_providers_) {
|
||||
results_.back().verification_map[provider] = Disposition::kNotRun;
|
||||
}
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
bool SymmOperationProfiler::verify_cutlass(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
if (!options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (options.execution_mode == ExecutionMode::kDryRun) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Initialize structure containing Symm arguments
|
||||
symm_workspace_.arguments.A = symm_workspace_.A->data();
|
||||
symm_workspace_.arguments.B = symm_workspace_.B->data();
|
||||
symm_workspace_.arguments.C = symm_workspace_.C->data();
|
||||
symm_workspace_.arguments.D = symm_workspace_.Computed->data();
|
||||
symm_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
symm_workspace_.arguments.beta = problem_.beta.data();
|
||||
symm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
//
|
||||
// Run the CUTLASS operation
|
||||
//
|
||||
|
||||
results_.back().status = operation->run(
|
||||
&symm_workspace_.arguments,
|
||||
symm_workspace_.host_workspace.data(),
|
||||
symm_workspace_.device_workspace.data());
|
||||
|
||||
if (results_.back().status != Status::kSuccess) {
|
||||
results_.back().disposition = Disposition::kFailed;
|
||||
return false;
|
||||
}
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
if (result != cudaSuccess) {
|
||||
results_.back().disposition = Disposition::kFailed;
|
||||
return false;
|
||||
}
|
||||
|
||||
// CUTLASS op ran the but not yet verified against any verification provider
|
||||
results_.back().disposition = Disposition::kNotVerified;
|
||||
|
||||
//
|
||||
// Run verification providers
|
||||
//
|
||||
|
||||
if (options.verification.enabled) {
|
||||
|
||||
#if CUTLASS_ENABLE_CUBLAS
|
||||
if (options.verification.provider_enabled(library::Provider::kCUBLAS)) {
|
||||
|
||||
// Guard against unsupported cases
|
||||
auto const & symm_desc = static_cast<library::SymmDescription const &>(operation->description());
|
||||
|
||||
if (cublas_satisfies(symm_desc) == Status::kSuccess) {
|
||||
|
||||
// call cublas verification if supported
|
||||
verify_with_cublas_(
|
||||
options,
|
||||
report,
|
||||
device_context,
|
||||
operation,
|
||||
problem_space,
|
||||
problem);
|
||||
}
|
||||
|
||||
else {
|
||||
// set verification map for cublas to not supported
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kNotSupported;
|
||||
}
|
||||
}
|
||||
#endif // #if CUTLASS_ENABLE_CUBLAS
|
||||
|
||||
// Update disposition to worst case verification outcome among all
|
||||
// verification providers which are supported
|
||||
bool is_any_verification_run_passed = false;
|
||||
for(auto &m : results_.back().verification_map) {
|
||||
if(m.second == Disposition::kFailed || m.second == Disposition::kIncorrect) {
|
||||
results_.back().disposition = m.second;
|
||||
return true;
|
||||
}
|
||||
if(!is_any_verification_run_passed && m.second == Disposition::kPassed) {
|
||||
is_any_verification_run_passed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if(is_any_verification_run_passed) {
|
||||
results_.back().disposition = Disposition::kPassed;
|
||||
}
|
||||
}
|
||||
|
||||
// Return true means continue profiling
|
||||
return true;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
bool SymmOperationProfiler::verify_with_cublas_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
|
||||
#if CUTLASS_ENABLE_CUBLAS
|
||||
|
||||
library::SymmDescription const &symm_desc =
|
||||
static_cast<library::SymmDescription const &>(operation->description());
|
||||
|
||||
//
|
||||
// Construct cuBLAS operators
|
||||
//
|
||||
|
||||
CublasCreate handle;
|
||||
cublasStatus_t status = handle.get_cublas_create_status();
|
||||
|
||||
if (status != CUBLAS_STATUS_SUCCESS) {
|
||||
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// Initialize state
|
||||
//
|
||||
|
||||
try {
|
||||
|
||||
//
|
||||
// Construct dispatcher to cublas<t>Symm()
|
||||
//
|
||||
|
||||
// Initialize structure containing Symm arguments
|
||||
symm_workspace_.arguments.A = symm_workspace_.A->data();
|
||||
symm_workspace_.arguments.B = symm_workspace_.B->data();
|
||||
symm_workspace_.arguments.C = symm_workspace_.Reference->data();
|
||||
symm_workspace_.arguments.D = symm_workspace_.Reference->data();
|
||||
symm_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
symm_workspace_.arguments.beta = problem_.beta.data();
|
||||
symm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
detail::cublasSymmDispatcher symm_op(
|
||||
symm_desc,
|
||||
symm_workspace_.configuration,
|
||||
symm_workspace_.arguments
|
||||
);
|
||||
|
||||
if (symm_op.status != Status::kSuccess) {
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kNotRun;
|
||||
return true;
|
||||
}
|
||||
|
||||
results_.back().status = Status::kSuccess;
|
||||
|
||||
status = symm_op(handle);
|
||||
|
||||
// Handle errors
|
||||
if (status != CUBLAS_STATUS_SUCCESS) {
|
||||
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// Verify results
|
||||
//
|
||||
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = compare_tensors(
|
||||
options,
|
||||
*symm_workspace_.Computed,
|
||||
*symm_workspace_.Reference
|
||||
);
|
||||
|
||||
// Save workspace if incorrect
|
||||
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] == Disposition::kIncorrect) {
|
||||
|
||||
save_workspace(
|
||||
device_context,
|
||||
options,
|
||||
symm_desc,
|
||||
library::Provider::kCUTLASS,
|
||||
library::Provider::kCUBLAS);
|
||||
}
|
||||
}
|
||||
catch (...) {
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
// Return true means continue profiling
|
||||
return true;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Measures performance results
|
||||
bool SymmOperationProfiler::profile(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
|
||||
|
||||
// Initialize structure containing Symm arguments
|
||||
symm_workspace_.arguments.A = symm_workspace_.A->data();
|
||||
symm_workspace_.arguments.B = symm_workspace_.B->data();
|
||||
symm_workspace_.arguments.C = symm_workspace_.C->data();
|
||||
symm_workspace_.arguments.D = symm_workspace_.Computed->data();
|
||||
symm_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
symm_workspace_.arguments.beta = problem_.beta.data();
|
||||
symm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
results_.back().status = profile_cutlass_(
|
||||
results_.back().runtime,
|
||||
options,
|
||||
operation,
|
||||
&symm_workspace_.arguments,
|
||||
symm_workspace_.host_workspace.data(),
|
||||
symm_workspace_.device_workspace.data()
|
||||
);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,230 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Defines a math function
|
||||
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <algorithm>
|
||||
#include <unordered_map>
|
||||
|
||||
// CUTLASS Library includes
|
||||
#include "cutlass/blas3.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/util.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
// Profiler includes
|
||||
#include "options.h"
|
||||
#include "device_context.h"
|
||||
#include "operation_profiler.h"
|
||||
#include "performance_result.h"
|
||||
#include "problem_space.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
/// Abstract base class for each math function
|
||||
class SymmOperationProfiler : public OperationProfiler {
|
||||
public:
|
||||
|
||||
/// Problem structure obtained from problem space
|
||||
struct SymmProblem {
|
||||
int64_t m;
|
||||
int64_t n;
|
||||
int64_t lda;
|
||||
int64_t ldb;
|
||||
int64_t ldc;
|
||||
SideMode side_mode;
|
||||
FillMode fill_mode;
|
||||
BlasMode blas_mode;
|
||||
std::vector<uint8_t> alpha;
|
||||
std::vector<uint8_t> beta;
|
||||
int64_t split_k_slices;
|
||||
int64_t batch_count;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
SymmProblem():
|
||||
m(16), n(16), lda(0), ldb(0), ldc(0),
|
||||
side_mode(SideMode::kInvalid), fill_mode(FillMode::kInvalid), blas_mode(BlasMode::kInvalid),
|
||||
split_k_slices(1), batch_count(1) { }
|
||||
|
||||
/// Parses the problem
|
||||
Status parse(
|
||||
library::SymmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Total number of bytes loaded
|
||||
int64_t bytes(library::SymmDescription const &operation_desc) const;
|
||||
|
||||
/// Total number of flops computed
|
||||
int64_t flops(library::SymmDescription const &operation_desc) const;
|
||||
|
||||
/// Initializes a performance result
|
||||
void initialize_result(
|
||||
PerformanceResult &result,
|
||||
library::SymmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space);
|
||||
};
|
||||
|
||||
/// Workspace used
|
||||
struct SymmWorkspace {
|
||||
|
||||
DeviceAllocation *A;
|
||||
DeviceAllocation *B;
|
||||
DeviceAllocation *C;
|
||||
DeviceAllocation *Computed;
|
||||
DeviceAllocation *Reference;
|
||||
|
||||
library::SymmConfiguration configuration;
|
||||
library::SymmArguments arguments;
|
||||
|
||||
/// Buffer used for the operation's host workspace
|
||||
std::vector<uint8_t> host_workspace;
|
||||
|
||||
/// Buffer used for the operations' device workspace
|
||||
DeviceAllocation device_workspace;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
SymmWorkspace():
|
||||
A(nullptr), B(nullptr), C(nullptr), Computed(nullptr), Reference(nullptr) { }
|
||||
};
|
||||
|
||||
protected:
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// GEMM problem obtained from problem space
|
||||
SymmProblem problem_;
|
||||
|
||||
/// Device memory allocations
|
||||
SymmWorkspace symm_workspace_;
|
||||
|
||||
|
||||
public:
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Ctor
|
||||
SymmOperationProfiler(Options const &options);
|
||||
|
||||
/// Destructor
|
||||
virtual ~SymmOperationProfiler();
|
||||
|
||||
/// Prints usage statement for the math function
|
||||
virtual void print_usage(std::ostream &out) const;
|
||||
|
||||
/// Prints examples
|
||||
virtual void print_examples(std::ostream &out) const;
|
||||
|
||||
/// Extracts the problem dimensions
|
||||
virtual Status initialize_configuration(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Initializes workspace
|
||||
virtual Status initialize_workspace(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
virtual bool verify_cutlass(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Measures performance results
|
||||
virtual bool profile(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
protected:
|
||||
|
||||
/// Initializes the performance result
|
||||
void initialize_result_(
|
||||
PerformanceResult &result,
|
||||
Options const &options,
|
||||
library::SymmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space);
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
bool verify_with_cublas_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,704 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Execution environment
|
||||
|
||||
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
#include <iomanip>
|
||||
#include <ios>
|
||||
|
||||
#include "cutlass/core_io.h"
|
||||
|
||||
#include "cublas_helpers.h"
|
||||
#include "trmm_operation_profiler.h"
|
||||
#include "gpu_timer.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Ctor
|
||||
TrmmOperationProfiler::TrmmOperationProfiler(Options const &options):
|
||||
OperationProfiler(
|
||||
options,
|
||||
library::OperationKind::kTrmm,
|
||||
{
|
||||
{ArgumentTypeID::kEnumerated, {"trmm_kind"}, "Variant of TRMM (universal)"},
|
||||
{ArgumentTypeID::kInteger, {"m", "problem-size::m"}, "M dimension of the TRMM problem space"},
|
||||
{ArgumentTypeID::kInteger, {"n", "problem-size::n"}, "N dimension of the TRMM problem space"},
|
||||
{ArgumentTypeID::kTensor, {"A"}, "Tensor storing the A operand"},
|
||||
{ArgumentTypeID::kEnumerated, {"side_mode"}, "Side Mode for TRMM (left, right)"},
|
||||
{ArgumentTypeID::kEnumerated, {"fill_mode"}, "Fill Mode for TRMM (lower, upper)"},
|
||||
{ArgumentTypeID::kEnumerated, {"diag_type"}, "Diag Type for TRMM (nonunit, unit)"},
|
||||
{ArgumentTypeID::kTensor, {"B"}, "Tensor storing the B operand"},
|
||||
{ArgumentTypeID::kTensor, {"D"}, "Tensor storing the D operand"},
|
||||
{ArgumentTypeID::kScalar, {"alpha", "epilogue::alpha"}, "Epilogue scalar alpha"},
|
||||
{ArgumentTypeID::kScalar, {"beta", "epilogue::beta"}, "Epilogue scalar beta"},
|
||||
{ArgumentTypeID::kInteger, {"split_k_slices", "split-k-slices"}, "Number of partitions of K dimension"},
|
||||
{ArgumentTypeID::kInteger, {"batch_count", "batch-count"}, "Number of TRMMs computed in one batch"},
|
||||
},
|
||||
{ library::Provider::kCUBLAS}
|
||||
) {
|
||||
description_ = " Triangular Matrix-Multiplication. D = alpha * A * B or alpha * B * A";
|
||||
}
|
||||
|
||||
/// Destructor
|
||||
TrmmOperationProfiler::~TrmmOperationProfiler() {
|
||||
|
||||
}
|
||||
|
||||
/// Prints usage statement for the math function
|
||||
void TrmmOperationProfiler::print_usage(std::ostream &out) const {
|
||||
out << "TRMM" << "\n\n";
|
||||
|
||||
OperationProfiler::print_usage(out);
|
||||
}
|
||||
|
||||
/// Prints examples
|
||||
void TrmmOperationProfiler::print_examples(std::ostream &out) const {
|
||||
|
||||
out << "\nExamples:\n\n"
|
||||
<< "Profile a particular problem size:\n"
|
||||
<< " $ cutlass_profiler --operation=Trmm --n=1024 --m=128\n\n"
|
||||
|
||||
<< "Schmoo over problem size and beta:\n"
|
||||
<< " $ cutlass_profiler --operation=Trmm --n=1024:4096:256 --m=128:8192:128 --beta=0,1,2.5\n\n"
|
||||
|
||||
<< "Schmoo over accumulator types:\n"
|
||||
<< " $ cutlass_profiler --operation=Trmm --accumulator-type=f16,f32\n\n"
|
||||
|
||||
<< "Run when A is f16 with column-major or A is any datatype with row-major (For column major, use column, col, or n. For row major use, row or t):\n"
|
||||
<< " $ cutlass_profiler --operation=Trmm --A=f16:column or --A=*:row\n\n"
|
||||
|
||||
<< "Using various input value distribution:\n"
|
||||
<< " $ cutlass_profiler --operation=Trmm --dist=uniform,min:0,max:3\n"
|
||||
<< " $ cutlass_profiler --operation=Trmm --dist=gaussian,mean:0,stddev:3\n"
|
||||
<< " $ cutlass_profiler --operation=Trmm --dist=sequential,start:0,delta:1\n\n"
|
||||
|
||||
<< "Run a kernel with cta tile size of 256x128x32 and save workspace if results are incorrect (note that --cta-tile::k=32 is default cta-tile size):\n"
|
||||
<< " $ cutlass_profiler --operation=Trmm --cta_m=256 --cta_n=128 --cta_k=32 --save-workspace=incorrect\n\n"
|
||||
|
||||
<< "Test your changes to trmm kernels with a quick functional test and save results in functional-test.csv:\n"
|
||||
<< " $ cutlass_profiler --operation=Trmm \\ \n"
|
||||
<< " --n=8,56,120,136,256,264,512,520,1024,1032,4096,8192,16384 \\ \n"
|
||||
<< " --k=8,16,32,64,128,256,288,384,504,512,520 \\ \n"
|
||||
<< " --beta=0,1,2 --profiling-iterations=1 \\ \n"
|
||||
<< " --providers=cutlass --output=functional-test.csv\n\n";
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if 0
|
||||
// used this for debugging
|
||||
static std::string byte_string(std::vector<uint8_t> const &bytes) {
|
||||
std::stringstream ss;
|
||||
|
||||
ss << "0x";
|
||||
|
||||
for (size_t idx = bytes.size(); idx > 0; --idx) {
|
||||
ss << std::hex << std::setw(2) << std::setfill('0') << uint32_t(bytes.at(idx - 1));
|
||||
}
|
||||
|
||||
return ss.str();
|
||||
}
|
||||
#endif
|
||||
|
||||
Status TrmmOperationProfiler::TrmmProblem::parse(
|
||||
library::TrmmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
if (!arg_as_int(this->m, "m", problem_space, problem)) {
|
||||
// default value
|
||||
this->m = 1024;
|
||||
}
|
||||
|
||||
if (!arg_as_int(this->n, "n", problem_space, problem)) {
|
||||
// default value
|
||||
this->n = 1024;
|
||||
}
|
||||
|
||||
if (!arg_as_int(this->split_k_slices, "split_k_slices", problem_space, problem)) {
|
||||
// default value
|
||||
this->split_k_slices = 1;
|
||||
}
|
||||
|
||||
if (!arg_as_int(this->batch_count, "batch_count", problem_space, problem)) {
|
||||
// default value
|
||||
this->batch_count = 1;
|
||||
}
|
||||
|
||||
if (this->split_k_slices > 1 && this->batch_count > 1) {
|
||||
// At least one of these must be one
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!tensor_description_satisfies(operation_desc.A, "A", problem_space, problem)) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!tensor_description_satisfies(operation_desc.B, "B", problem_space, problem)) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!tensor_description_satisfies(operation_desc.D, "D", problem_space, problem)) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!arg_as_scalar(
|
||||
this->alpha,
|
||||
operation_desc.element_epilogue,
|
||||
"alpha",
|
||||
problem_space,
|
||||
problem)) {
|
||||
|
||||
if (!cast_from_double(this->alpha, operation_desc.element_epilogue, 1)) {
|
||||
return Status::kErrorInternal;
|
||||
}
|
||||
}
|
||||
|
||||
if (!arg_as_scalar(
|
||||
this->beta,
|
||||
operation_desc.element_epilogue,
|
||||
"beta",
|
||||
problem_space,
|
||||
problem)) {
|
||||
|
||||
if (!cast_from_double(this->beta, operation_desc.element_epilogue, 0)) {
|
||||
return Status::kErrorInternal;
|
||||
}
|
||||
}
|
||||
|
||||
if (operation_desc.side_mode == SideMode::kLeft) {
|
||||
this->lda = DeviceAllocation::get_packed_layout(
|
||||
operation_desc.A.layout, {int(this->m), int(this->m)}).front();
|
||||
}
|
||||
else if (operation_desc.side_mode == SideMode::kRight) {
|
||||
this->lda = DeviceAllocation::get_packed_layout(
|
||||
operation_desc.A.layout, {int(this->n), int(this->n)}).front();
|
||||
}
|
||||
|
||||
this->ldb = DeviceAllocation::get_packed_layout(
|
||||
operation_desc.B.layout, {int(this->m), int(this->n)}).front();
|
||||
|
||||
this->ldd = DeviceAllocation::get_packed_layout(
|
||||
operation_desc.D.layout, {int(this->m), int(this->n)}).front();
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/// Initializes a performance result
|
||||
void TrmmOperationProfiler::TrmmProblem::initialize_result(
|
||||
PerformanceResult &result,
|
||||
library::TrmmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space) {
|
||||
|
||||
result.arguments.resize(problem_space.rank());
|
||||
|
||||
set_argument(result, "trmm_kind", problem_space, library::to_string(operation_desc.trmm_kind));
|
||||
|
||||
set_argument(result, "A", problem_space,
|
||||
std::string(library::to_string(operation_desc.A.element)) + ":" + library::to_string(operation_desc.A.layout));
|
||||
|
||||
set_argument(result, "side_mode", problem_space, library::to_string(operation_desc.side_mode));
|
||||
|
||||
set_argument(result, "fill_mode", problem_space, library::to_string(operation_desc.fill_mode));
|
||||
|
||||
set_argument(result, "diag_type", problem_space, library::to_string(operation_desc.diag_type));
|
||||
|
||||
set_argument(result, "B", problem_space,
|
||||
std::string(library::to_string(operation_desc.B.element)) + ":" + library::to_string(operation_desc.B.layout));
|
||||
|
||||
set_argument(result, "D", problem_space,
|
||||
std::string(library::to_string(operation_desc.D.element)) + ":" + library::to_string(operation_desc.D.layout));
|
||||
|
||||
set_argument(result, "m", problem_space, m);
|
||||
set_argument(result, "n", problem_space, n);
|
||||
|
||||
set_argument(result, "split_k_slices", problem_space, split_k_slices);
|
||||
set_argument(result, "batch_count", problem_space, batch_count);
|
||||
|
||||
set_argument(result, "alpha", problem_space,
|
||||
library::lexical_cast(alpha, operation_desc.element_epilogue));
|
||||
|
||||
set_argument(result, "beta", problem_space,
|
||||
library::lexical_cast(beta, operation_desc.element_epilogue));
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Extracts the problem dimensions
|
||||
Status TrmmOperationProfiler::initialize_configuration(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
library::TrmmDescription const &operation_desc =
|
||||
static_cast<library::TrmmDescription const &>(operation->description());
|
||||
|
||||
if (operation_desc.trmm_kind != library::TrmmKind::kUniversal) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
Status status = problem_.parse(operation_desc, problem_space, problem);
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
trmm_workspace_.configuration.problem_size.m() = int(problem_.m);
|
||||
trmm_workspace_.configuration.problem_size.n() = int(problem_.n);
|
||||
trmm_workspace_.configuration.problem_size.k() = (operation_desc.side_mode == SideMode::kLeft)
|
||||
? int(problem_.m) : int(problem_.n);
|
||||
trmm_workspace_.configuration.lda = problem_.lda;
|
||||
trmm_workspace_.configuration.ldb = problem_.ldb;
|
||||
trmm_workspace_.configuration.ldd = problem_.ldd;
|
||||
//trmm_workspace_.configuration.split_k_slices = int(problem_.split_k_slices);
|
||||
trmm_workspace_.configuration.batch_count = int(problem_.split_k_slices);
|
||||
|
||||
trmm_workspace_.arguments.A = nullptr;
|
||||
trmm_workspace_.arguments.B = nullptr;
|
||||
trmm_workspace_.arguments.D = nullptr;
|
||||
trmm_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
trmm_workspace_.arguments.beta = problem_.beta.data();
|
||||
trmm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
initialize_result_(this->model_result_, options, operation_desc, problem_space);
|
||||
|
||||
return operation->can_implement(&trmm_workspace_.configuration, &trmm_workspace_.arguments);
|
||||
}
|
||||
|
||||
/// Initializes the performance result
|
||||
void TrmmOperationProfiler::initialize_result_(
|
||||
PerformanceResult &result,
|
||||
Options const &options,
|
||||
library::TrmmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space) {
|
||||
|
||||
result.provider = library::Provider::kCUTLASS;
|
||||
result.disposition = Disposition::kNotRun;
|
||||
result.status = Status::kSuccess;
|
||||
result.operation_name = operation_desc.name;
|
||||
|
||||
problem_.initialize_result(result, operation_desc, problem_space);
|
||||
|
||||
OperationProfiler::initialize_result_(result, operation_desc, problem_space);
|
||||
|
||||
if (operation_desc.side_mode == SideMode::kLeft) {
|
||||
// Input bytes read and Output bytes written for the trmm problem
|
||||
result.bytes =
|
||||
// Half matrix including the diagonal will have (M*(M+1))/2 elements
|
||||
int64_t(library::sizeof_bits(operation_desc.A.element) * problem_.m / 8) * (problem_.m + 1) / 2 +
|
||||
int64_t(library::sizeof_bits(operation_desc.B.element) * problem_.m / 8) * problem_.n +
|
||||
int64_t(library::sizeof_bits(operation_desc.D.element) * problem_.m / 8) * problem_.n;
|
||||
} else if (operation_desc.side_mode == SideMode::kRight) {
|
||||
// Input bytes read and Output bytes written for the trmm problem
|
||||
result.bytes =
|
||||
// Half matrix including the diagonal will have (N*(N+1))/2 elements
|
||||
int64_t(library::sizeof_bits(operation_desc.A.element) * problem_.n / 8) * (problem_.n + 1) / 2 +
|
||||
int64_t(library::sizeof_bits(operation_desc.B.element) * problem_.m / 8) * problem_.n +
|
||||
int64_t(library::sizeof_bits(operation_desc.D.element) * problem_.m / 8) * problem_.n;
|
||||
}
|
||||
|
||||
// FLOPs = 2 * [ ( M * (M+1)/2 * N ) ] // Beta is zero
|
||||
result.flops = problem_.m * (problem_.m + 1) * problem_.n;
|
||||
|
||||
result.runtime = 0;
|
||||
|
||||
// complex-valued support
|
||||
switch (operation_desc.tile_description.math_instruction.math_operation) {
|
||||
case library::MathOperationID::kMultiplyAddComplex:
|
||||
result.flops *= 4;
|
||||
break;
|
||||
|
||||
case library::MathOperationID::kMultiplyAddComplexFastF32:
|
||||
result.flops *= 4;
|
||||
break;
|
||||
|
||||
default: break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/// Initializes workspace
|
||||
Status TrmmOperationProfiler::initialize_workspace(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
library::TrmmDescription const &operation_desc =
|
||||
static_cast<library::TrmmDescription const &>(operation->description());
|
||||
|
||||
if (options.execution_mode != ExecutionMode::kDryRun) {
|
||||
|
||||
if (operation_desc.side_mode == SideMode::kLeft) {
|
||||
trmm_workspace_.A = device_context.allocate_tensor(
|
||||
options,
|
||||
"A",
|
||||
operation_desc.A.element,
|
||||
operation_desc.A.layout,
|
||||
{int(problem_.m), int(problem_.m)},
|
||||
{int(problem_.lda)},
|
||||
1 // batch_count = 1, default
|
||||
);
|
||||
} else if (operation_desc.side_mode == SideMode::kRight) {
|
||||
trmm_workspace_.A = device_context.allocate_tensor(
|
||||
options,
|
||||
"A",
|
||||
operation_desc.A.element,
|
||||
operation_desc.A.layout,
|
||||
{int(problem_.n), int(problem_.n)},
|
||||
{int(problem_.lda)},
|
||||
1 // batch_count = 1, default
|
||||
);
|
||||
}
|
||||
|
||||
trmm_workspace_.B = device_context.allocate_tensor(
|
||||
options,
|
||||
"B",
|
||||
operation_desc.B.element,
|
||||
operation_desc.B.layout,
|
||||
{int(problem_.m), int(problem_.n)},
|
||||
{int(problem_.ldb)}
|
||||
);
|
||||
|
||||
trmm_workspace_.Computed = device_context.allocate_tensor(
|
||||
"D",
|
||||
operation_desc.D.element,
|
||||
operation_desc.D.layout,
|
||||
{int(problem_.m), int(problem_.n)},
|
||||
{int(problem_.ldd)}
|
||||
);
|
||||
|
||||
trmm_workspace_.Reference = device_context.allocate_tensor(
|
||||
"Reference",
|
||||
operation_desc.D.element,
|
||||
operation_desc.D.layout,
|
||||
{int(problem_.m), int(problem_.n)},
|
||||
{int(problem_.ldd)}
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
//
|
||||
// Initialize the CUTLASS operation
|
||||
//
|
||||
Status status = Status::kSuccess;
|
||||
|
||||
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
|
||||
|
||||
if (options.execution_mode != ExecutionMode::kDryRun) {
|
||||
|
||||
uint64_t workspace_size = operation->get_host_workspace_size(&trmm_workspace_.configuration);
|
||||
trmm_workspace_.host_workspace.resize(workspace_size, 0);
|
||||
|
||||
workspace_size = operation->get_device_workspace_size(&trmm_workspace_.configuration);
|
||||
trmm_workspace_.device_workspace.reset(library::NumericTypeID::kU8, workspace_size);
|
||||
|
||||
status = operation->initialize(
|
||||
&trmm_workspace_.configuration,
|
||||
trmm_workspace_.host_workspace.data(),
|
||||
trmm_workspace_.device_workspace.data());
|
||||
}
|
||||
|
||||
//
|
||||
// If CUTLASS is enabled, generate a result for it
|
||||
//
|
||||
results_.push_back(model_result_);
|
||||
results_.back().provider = library::Provider::kCUTLASS;
|
||||
results_.back().op_kind = library::OperationKind::kTrmm;
|
||||
results_.back().disposition = Disposition::kNotRun;
|
||||
|
||||
for(auto provider : verification_providers_) {
|
||||
results_.back().verification_map[provider] = Disposition::kNotRun;
|
||||
}
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
bool TrmmOperationProfiler::verify_cutlass(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
if (!options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (options.execution_mode == ExecutionMode::kDryRun) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Initialize structure containing TRMM arguments
|
||||
trmm_workspace_.arguments.A = trmm_workspace_.A->data();
|
||||
trmm_workspace_.arguments.B = trmm_workspace_.B->data();
|
||||
trmm_workspace_.arguments.D = trmm_workspace_.Computed->data();
|
||||
trmm_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
trmm_workspace_.arguments.beta = problem_.beta.data();
|
||||
trmm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
//
|
||||
// Run the CUTLASS operation
|
||||
//
|
||||
|
||||
results_.back().status = operation->run(
|
||||
&trmm_workspace_.arguments,
|
||||
trmm_workspace_.host_workspace.data(),
|
||||
trmm_workspace_.device_workspace.data());
|
||||
|
||||
if (results_.back().status != Status::kSuccess) {
|
||||
results_.back().disposition = Disposition::kFailed;
|
||||
return false;
|
||||
}
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
if (result != cudaSuccess) {
|
||||
results_.back().disposition = Disposition::kFailed;
|
||||
return false;
|
||||
}
|
||||
|
||||
// CUTLASS op ran the but not yet verified against any verification provider
|
||||
results_.back().disposition = Disposition::kNotVerified;
|
||||
|
||||
//
|
||||
// Run verification providers
|
||||
//
|
||||
|
||||
if (options.verification.enabled) {
|
||||
|
||||
#if CUTLASS_ENABLE_CUBLAS
|
||||
if (options.verification.provider_enabled(library::Provider::kCUBLAS)) {
|
||||
|
||||
// Guard against unsupported cases
|
||||
auto const & trmm_desc = static_cast<library::TrmmDescription const &>(operation->description());
|
||||
|
||||
if (cublas_satisfies(trmm_desc) == Status::kSuccess) {
|
||||
|
||||
// call cublas verification if supported
|
||||
verify_with_cublas_(
|
||||
options,
|
||||
report,
|
||||
device_context,
|
||||
operation,
|
||||
problem_space,
|
||||
problem);
|
||||
}
|
||||
|
||||
else {
|
||||
// set verification map for cublas to not supported
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kNotSupported;
|
||||
}
|
||||
}
|
||||
#endif // #if CUTLASS_ENABLE_CUBLAS
|
||||
|
||||
// Update disposition to worst case verification outcome among all
|
||||
// verification providers which are supported
|
||||
bool is_any_verification_run_passed = false;
|
||||
for(auto &m : results_.back().verification_map) {
|
||||
if(m.second == Disposition::kFailed || m.second == Disposition::kIncorrect) {
|
||||
results_.back().disposition = m.second;
|
||||
return true;
|
||||
}
|
||||
if(!is_any_verification_run_passed && m.second == Disposition::kPassed) {
|
||||
is_any_verification_run_passed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if(is_any_verification_run_passed) {
|
||||
results_.back().disposition = Disposition::kPassed;
|
||||
}
|
||||
}
|
||||
|
||||
// Return true means continue profiling
|
||||
return true;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
bool TrmmOperationProfiler::verify_with_cublas_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
|
||||
#if CUTLASS_ENABLE_CUBLAS
|
||||
|
||||
library::TrmmDescription const &trmm_desc =
|
||||
static_cast<library::TrmmDescription const &>(operation->description());
|
||||
|
||||
//
|
||||
// Construct cuBLAS operators
|
||||
//
|
||||
|
||||
CublasCreate handle;
|
||||
cublasStatus_t status = handle.get_cublas_create_status();
|
||||
|
||||
if (status != CUBLAS_STATUS_SUCCESS) {
|
||||
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// Initialize state
|
||||
//
|
||||
|
||||
try {
|
||||
|
||||
//
|
||||
// Construct dispatcher to cublas<t>Trmm()
|
||||
//
|
||||
|
||||
// Initialize structure containing TRMM arguments
|
||||
trmm_workspace_.arguments.A = trmm_workspace_.A->data();
|
||||
trmm_workspace_.arguments.B = trmm_workspace_.B->data();
|
||||
trmm_workspace_.arguments.D = trmm_workspace_.Reference->data();
|
||||
trmm_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
trmm_workspace_.arguments.beta = problem_.beta.data();
|
||||
trmm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
detail::cublasTrmmDispatcher trmm_op(
|
||||
trmm_desc,
|
||||
trmm_workspace_.configuration,
|
||||
trmm_workspace_.arguments
|
||||
);
|
||||
|
||||
if (trmm_op.status != Status::kSuccess) {
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kNotRun;
|
||||
return true;
|
||||
}
|
||||
|
||||
results_.back().status = Status::kSuccess;
|
||||
|
||||
status = trmm_op(handle);
|
||||
|
||||
// Handle errors
|
||||
if (status != CUBLAS_STATUS_SUCCESS) {
|
||||
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// Verify results
|
||||
//
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = compare_tensors(
|
||||
options,
|
||||
*trmm_workspace_.Computed,
|
||||
*trmm_workspace_.Reference
|
||||
);
|
||||
|
||||
// Save workspace if incorrect
|
||||
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] == Disposition::kIncorrect) {
|
||||
|
||||
save_workspace(
|
||||
device_context,
|
||||
options,
|
||||
trmm_desc,
|
||||
library::Provider::kCUTLASS,
|
||||
library::Provider::kCUBLAS);
|
||||
}
|
||||
}
|
||||
catch (...) {
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
// Return true means continue profiling
|
||||
return true;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Measures performance results
|
||||
bool TrmmOperationProfiler::profile(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
|
||||
|
||||
// Initialize structure containing TRMM arguments
|
||||
trmm_workspace_.arguments.A = trmm_workspace_.A->data();
|
||||
trmm_workspace_.arguments.B = trmm_workspace_.B->data();
|
||||
trmm_workspace_.arguments.D = trmm_workspace_.Computed->data();
|
||||
trmm_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
trmm_workspace_.arguments.beta = problem_.beta.data();
|
||||
trmm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
results_.back().status = profile_cutlass_(
|
||||
results_.back().runtime,
|
||||
options,
|
||||
operation,
|
||||
&trmm_workspace_.arguments,
|
||||
trmm_workspace_.host_workspace.data(),
|
||||
trmm_workspace_.device_workspace.data()
|
||||
);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,222 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Defines a math function
|
||||
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <algorithm>
|
||||
#include <unordered_map>
|
||||
|
||||
// CUTLASS Library includes
|
||||
#include "cutlass/blas3.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/util.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
// Profiler includes
|
||||
#include "options.h"
|
||||
#include "device_context.h"
|
||||
#include "operation_profiler.h"
|
||||
#include "performance_result.h"
|
||||
#include "problem_space.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Abstract base class for each math function
|
||||
class TrmmOperationProfiler : public OperationProfiler {
|
||||
public:
|
||||
|
||||
/// Problem structure obtained from problem space
|
||||
struct TrmmProblem {
|
||||
int64_t m;
|
||||
int64_t n;
|
||||
int64_t lda;
|
||||
int64_t ldb;
|
||||
int64_t ldd;
|
||||
SideMode side_mode;
|
||||
FillMode fill_mode;
|
||||
DiagType diag_type;
|
||||
std::vector<uint8_t> alpha;
|
||||
std::vector<uint8_t> beta;
|
||||
int64_t split_k_slices;
|
||||
int64_t batch_count;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
TrmmProblem():
|
||||
m(16), n(16), lda(0), ldb(0), ldd(0), split_k_slices(1), batch_count(1) { }
|
||||
|
||||
/// Parses the problem
|
||||
Status parse(
|
||||
library::TrmmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Initializes a performance result
|
||||
void initialize_result(
|
||||
PerformanceResult &result,
|
||||
library::TrmmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space);
|
||||
};
|
||||
|
||||
/// Workspace used
|
||||
struct TrmmWorkspace {
|
||||
|
||||
DeviceAllocation *A;
|
||||
DeviceAllocation *B;
|
||||
DeviceAllocation *D;
|
||||
DeviceAllocation *Computed;
|
||||
DeviceAllocation *Reference;
|
||||
|
||||
library::TrmmConfiguration configuration;
|
||||
library::TrmmArguments arguments;
|
||||
|
||||
/// Buffer used for the operation's host workspace
|
||||
std::vector<uint8_t> host_workspace;
|
||||
|
||||
/// Buffer used for the operations' device workspace
|
||||
DeviceAllocation device_workspace;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
TrmmWorkspace():
|
||||
A(nullptr), B(nullptr), D(nullptr), Computed(nullptr), Reference(nullptr) { }
|
||||
};
|
||||
|
||||
protected:
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// GEMM problem obtained from problem space
|
||||
TrmmProblem problem_;
|
||||
|
||||
/// Device memory allocations
|
||||
TrmmWorkspace trmm_workspace_;
|
||||
|
||||
|
||||
public:
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Ctor
|
||||
TrmmOperationProfiler(Options const &options);
|
||||
|
||||
/// Destructor
|
||||
virtual ~TrmmOperationProfiler();
|
||||
|
||||
/// Prints usage statement for the math function
|
||||
virtual void print_usage(std::ostream &out) const;
|
||||
|
||||
/// Prints examples
|
||||
virtual void print_examples(std::ostream &out) const;
|
||||
|
||||
/// Extracts the problem dimensions
|
||||
virtual Status initialize_configuration(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Initializes workspace
|
||||
virtual Status initialize_workspace(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
virtual bool verify_cutlass(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Measures performance results
|
||||
virtual bool profile(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
protected:
|
||||
|
||||
/// Initializes the performance result
|
||||
void initialize_result_(
|
||||
PerformanceResult &result,
|
||||
Options const &options,
|
||||
library::TrmmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space);
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
bool verify_with_cublas_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
Reference in New Issue
Block a user