leavelet/fmt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix bugs in utf8 decoder (#3056)

Browse Source 
Signed-off-by: Vladislav Shchapov <vladislav@shchapov.ru>

Signed-off-by: Vladislav Shchapov <vladislav@shchapov.ru>
This commit is contained in:
Vladislav Shchapov
2022-08-26 15:37:15 -07:00
committed by GitHub
parent 4a8e2949bb
commit f98048b621
3 changed files with 133 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/fmt/core.h
+7 -3
View File
@@ -2297,12 +2297,16 @@ constexpr auto to_ascii(Char c) -> underlying_t<Char> {
return c;
}
template <typename Char>
FMT_CONSTEXPR auto code_point_length_impl(Char begin) -> int {
return "\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\0\0\0\0\0\0\0\2\2\2\2\3\3\4"
[static_cast<unsigned char>(begin) >> 3];
}
template <typename Char>
FMT_CONSTEXPR auto code_point_length(const Char* begin) -> int {
if (const_check(sizeof(Char) != 1)) return 1;
auto lengths =
"\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\0\0\0\0\0\0\0\2\2\2\2\3\3\4";
int len = lengths[static_cast<unsigned char>(*begin) >> 3];
int len = code_point_length_impl(*begin);
// Compute the pointer to the next character early so that the next
// iteration can start working on the next character. Neither Clang
include/fmt/format.h
+11 -10
View File
@@ -602,25 +602,28 @@ FMT_CONSTEXPR FMT_NOINLINE auto copy_str_noinline(InputIt begin, InputIt end,
*/
FMT_CONSTEXPR inline auto utf8_decode(const char* s, uint32_t* c, int* e)
-> const char* {
constexpr const int prefix_masks[] = {0x00, 0x80, 0xe0, 0xf0, 0xf8};
constexpr const int masks[] = {0x00, 0x7f, 0x1f, 0x0f, 0x07};
constexpr const uint32_t mins[] = {4194304, 0, 128, 2048, 65536};
constexpr const int shiftc[] = {0, 18, 12, 6, 0};
constexpr const int shifte[] = {0, 6, 4, 2, 0};
int len = code_point_length(s);
const char* next = s + len;
int len = code_point_length_impl(*s);
// Compute the pointer to the next character early so that the next
// iteration can start working on the next character. Neither Clang
// nor GCC figure out this reordering on their own.
const char* next = s + len + !len;
using uchar = unsigned char;
// Assume a four-byte character and load four bytes. Unused bits are
// shifted out.
*c = uint32_t(s[0] & masks[len]) << 18;
*c |= uint32_t(s[1] & 0x3f) << 12;
*c |= uint32_t(s[2] & 0x3f) << 6;
*c |= uint32_t(s[3] & 0x3f) << 0;
*c = uint32_t(uchar(s[0]) & masks[len]) << 18;
*c |= uint32_t(uchar(s[1]) & 0x3f) << 12;
*c |= uint32_t(uchar(s[2]) & 0x3f) << 6;
*c |= uint32_t(uchar(s[3]) & 0x3f) << 0;
*c >>= shiftc[len];
// Accumulate the various error conditions.
using uchar = unsigned char;
*e = (*c < mins[len]) << 6; // non-canonical encoding
*e |= ((*c >> 11) == 0x1b) << 7; // surrogate half?
*e |= (*c > 0x10FFFF) << 8; // out of range?
@@ -629,8 +632,6 @@ FMT_CONSTEXPR inline auto utf8_decode(const char* s, uint32_t* c, int* e)
*e |= uchar(s[3]) >> 6;
*e ^= 0x2a; // top two bits of each tail byte correct?
*e >>= shifte[len];
*e |= ((uchar(s[0]) & prefix_masks[len]) !=
uchar((prefix_masks[len] << 1) & 0xFF)); // first byte correct?
return next;
}