From 44d3e17985befbd45457d5ad7f0a0387849e1b2f Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 21 Sep 2017 15:31:38 -0700 Subject: deps: ICU 60 bump - Update to released ICU 60.1, including: - CLDR 32 (many new languages and data improvements) - Unicode 10 (8,518 new characters, including four new scripts, 7,494 new Han characters, and 56 new emoji characters) - UTF-8 malformed bytes now handled according to W3C/WHATWG spec Fixes: https://github.com/nodejs/node/issues/15540 PR-URL: https://github.com/nodejs/node/pull/16876 Reviewed-By: James M Snell Reviewed-By: Michael Dawson --- deps/icu-small/source/common/unicode/utf8.h | 148 +++++++++++++++++----------- 1 file changed, 88 insertions(+), 60 deletions(-) (limited to 'deps/icu-small/source/common/unicode/utf8.h') diff --git a/deps/icu-small/source/common/unicode/utf8.h b/deps/icu-small/source/common/unicode/utf8.h index 9e56b50474..59b4b25570 100644 --- a/deps/icu-small/source/common/unicode/utf8.h +++ b/deps/icu-small/source/common/unicode/utf8.h @@ -41,34 +41,24 @@ /* internal definitions ----------------------------------------------------- */ - - /** * Counts the trail bytes for a UTF-8 lead byte. - * Returns 0 for 0..0xbf as well as for 0xfe and 0xff. + * Returns 0 for 0..0xc1 as well as for 0xf5..0xff. + * leadByte might be evaluated multiple times. * * This is internal since it is not meant to be called directly by external clients; * however it is called by public macros in this file and thus must remain stable. * - * Note: Beginning with ICU 50, the implementation uses a multi-condition expression - * which was shown in 2012 (on x86-64) to compile to fast, branch-free code. - * leadByte is evaluated multiple times. - * - * The pre-ICU 50 implementation used the exported array utf8_countTrailBytes: - * #define U8_COUNT_TRAIL_BYTES(leadByte) (utf8_countTrailBytes[leadByte]) - * leadByte was evaluated exactly once. - * * @param leadByte The first byte of a UTF-8 sequence. Must be 0..0xff. * @internal */ #define U8_COUNT_TRAIL_BYTES(leadByte) \ - ((uint8_t)(leadByte)<0xf0 ? \ - ((uint8_t)(leadByte)>=0xc0)+((uint8_t)(leadByte)>=0xe0) : \ - (uint8_t)(leadByte)<0xfe ? 3+((uint8_t)(leadByte)>=0xf8)+((uint8_t)(leadByte)>=0xfc) : 0) + (U8_IS_LEAD(leadByte) ? \ + ((uint8_t)(leadByte)>=0xe0)+((uint8_t)(leadByte)>=0xf0)+1 : 0) /** * Counts the trail bytes for a UTF-8 lead byte of a valid UTF-8 sequence. - * The maximum supported lead byte is 0xf4 corresponding to U+10FFFF. + * Returns 0 for 0..0xc1. Undefined for 0xf5..0xff. * leadByte might be evaluated multiple times. * * This is internal since it is not meant to be called directly by external clients; @@ -78,7 +68,7 @@ * @internal */ #define U8_COUNT_TRAIL_BYTES_UNSAFE(leadByte) \ - (((leadByte)>=0xc0)+((leadByte)>=0xe0)+((leadByte)>=0xf0)) + (((uint8_t)(leadByte)>=0xc2)+((uint8_t)(leadByte)>=0xe0)+((uint8_t)(leadByte)>=0xf0)) /** * Mask a UTF-8 lead byte, leave only the lower bits that form part of the code point value. @@ -89,6 +79,40 @@ */ #define U8_MASK_LEAD_BYTE(leadByte, countTrailBytes) ((leadByte)&=(1<<(6-(countTrailBytes)))-1) +/** + * Internal bit vector for 3-byte UTF-8 validity check, for use in U8_IS_VALID_LEAD3_AND_T1. + * Each bit indicates whether one lead byte + first trail byte pair starts a valid sequence. + * Lead byte E0..EF bits 3..0 are used as byte index, + * first trail byte bits 7..5 are used as bit index into that byte. + * @see U8_IS_VALID_LEAD3_AND_T1 + * @internal + */ +#define U8_LEAD3_T1_BITS "\x20\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x10\x30\x30" + +/** + * Internal 3-byte UTF-8 validity check. + * Non-zero if lead byte E0..EF and first trail byte 00..FF start a valid sequence. + * @internal + */ +#define U8_IS_VALID_LEAD3_AND_T1(lead, t1) (U8_LEAD3_T1_BITS[(lead)&0xf]&(1<<((uint8_t)(t1)>>5))) + +/** + * Internal bit vector for 4-byte UTF-8 validity check, for use in U8_IS_VALID_LEAD4_AND_T1. + * Each bit indicates whether one lead byte + first trail byte pair starts a valid sequence. + * First trail byte bits 7..4 are used as byte index, + * lead byte F0..F4 bits 2..0 are used as bit index into that byte. + * @see U8_IS_VALID_LEAD4_AND_T1 + * @internal + */ +#define U8_LEAD4_T1_BITS "\x00\x00\x00\x00\x00\x00\x00\x00\x1E\x0F\x0F\x0F\x00\x00\x00\x00" + +/** + * Internal 4-byte UTF-8 validity check. + * Non-zero if lead byte F0..F4 and first trail byte 00..FF start a valid sequence. + * @internal + */ +#define U8_IS_VALID_LEAD4_AND_T1(lead, t1) (U8_LEAD4_T1_BITS[(uint8_t)(t1)>>4]&(1<<((lead)&7))) + /** * Function for handling "next code point" with error-checking. * @@ -148,20 +172,21 @@ utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i); #define U8_IS_SINGLE(c) (((c)&0x80)==0) /** - * Is this code unit (byte) a UTF-8 lead byte? + * Is this code unit (byte) a UTF-8 lead byte? (0xC2..0xF4) * @param c 8-bit code unit (byte) * @return TRUE or FALSE * @stable ICU 2.4 */ -#define U8_IS_LEAD(c) ((uint8_t)((c)-0xc0)<0x3e) +#define U8_IS_LEAD(c) ((uint8_t)((c)-0xc2)<=0x32) +// 0x32=0xf4-0xc2 /** - * Is this code unit (byte) a UTF-8 trail byte? + * Is this code unit (byte) a UTF-8 trail byte? (0x80..0xBF) * @param c 8-bit code unit (byte) * @return TRUE or FALSE * @stable ICU 2.4 */ -#define U8_IS_TRAIL(c) (((c)&0xc0)==0x80) +#define U8_IS_TRAIL(c) ((int8_t)(c)<-0x40) /** * How many code units (bytes) are used for the UTF-8 encoding @@ -289,7 +314,7 @@ utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i); */ #define U8_NEXT_UNSAFE(s, i, c) { \ (c)=(uint8_t)(s)[(i)++]; \ - if((c)>=0x80) { \ + if(!U8_IS_SINGLE(c)) { \ if((c)<0xe0) { \ (c)=(((c)&0x1f)<<6)|((s)[(i)++]&0x3f); \ } else if((c)<0xf0) { \ @@ -325,22 +350,19 @@ utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i); */ #define U8_NEXT(s, i, length, c) { \ (c)=(uint8_t)(s)[(i)++]; \ - if((c)>=0x80) { \ + if(!U8_IS_SINGLE(c)) { \ uint8_t __t1, __t2; \ - if( /* handle U+1000..U+CFFF inline */ \ - (0xe0<(c) && (c)<=0xec) && \ - (((i)+1)<(length) || (length)<0) && \ - (__t1=(uint8_t)((s)[i]-0x80))<=0x3f && \ - (__t2=(uint8_t)((s)[(i)+1]-0x80))<= 0x3f \ - ) { \ - /* no need for (c&0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ \ - (c)=(UChar)(((c)<<12)|(__t1<<6)|__t2); \ + if( /* handle U+0800..U+FFFF inline */ \ + (0xe0<=(c) && (c)<0xf0) && \ + (((i)+1)<(length) || (length)<0) && \ + U8_IS_VALID_LEAD3_AND_T1((c), __t1=(s)[i]) && \ + (__t2=(s)[(i)+1]-0x80)<=0x3f) { \ + (c)=(((c)&0xf)<<12)|((__t1&0x3f)<<6)|__t2; \ (i)+=2; \ } else if( /* handle U+0080..U+07FF inline */ \ - ((c)<0xe0 && (c)>=0xc2) && \ - ((i)!=(length)) && \ - (__t1=(uint8_t)((s)[i]-0x80))<=0x3f \ - ) { \ + ((c)<0xe0 && (c)>=0xc2) && \ + ((i)!=(length)) && \ + (__t1=(s)[i]-0x80)<=0x3f) { \ (c)=(((c)&0x1f)<<6)|__t1; \ ++(i); \ } else { \ @@ -376,22 +398,19 @@ utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i); */ #define U8_NEXT_OR_FFFD(s, i, length, c) { \ (c)=(uint8_t)(s)[(i)++]; \ - if((c)>=0x80) { \ + if(!U8_IS_SINGLE(c)) { \ uint8_t __t1, __t2; \ - if( /* handle U+1000..U+CFFF inline */ \ - (0xe0<(c) && (c)<=0xec) && \ - (((i)+1)<(length) || (length)<0) && \ - (__t1=(uint8_t)((s)[i]-0x80))<=0x3f && \ - (__t2=(uint8_t)((s)[(i)+1]-0x80))<= 0x3f \ - ) { \ - /* no need for (c&0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ \ - (c)=(UChar)(((c)<<12)|(__t1<<6)|__t2); \ + if( /* handle U+0800..U+FFFF inline */ \ + (0xe0<=(c) && (c)<0xf0) && \ + (((i)+1)<(length) || (length)<0) && \ + U8_IS_VALID_LEAD3_AND_T1((c), __t1=(s)[i]) && \ + (__t2=(s)[(i)+1]-0x80)<=0x3f) { \ + (c)=(((c)&0xf)<<12)|((__t1&0x3f)<<6)|__t2; \ (i)+=2; \ } else if( /* handle U+0080..U+07FF inline */ \ - ((c)<0xe0 && (c)>=0xc2) && \ - ((i)!=(length)) && \ - (__t1=(uint8_t)((s)[i]-0x80))<=0x3f \ - ) { \ + ((c)<0xe0 && (c)>=0xc2) && \ + ((i)!=(length)) && \ + (__t1=(s)[i]-0x80)<=0x3f) { \ (c)=(((c)&0x1f)<<6)|__t1; \ ++(i); \ } else { \ @@ -476,7 +495,7 @@ utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i); * @stable ICU 2.4 */ #define U8_FWD_1_UNSAFE(s, i) { \ - (i)+=1+U8_COUNT_TRAIL_BYTES_UNSAFE((uint8_t)(s)[i]); \ + (i)+=1+U8_COUNT_TRAIL_BYTES_UNSAFE((s)[i]); \ } /** @@ -493,15 +512,24 @@ utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i); * @stable ICU 2.4 */ #define U8_FWD_1(s, i, length) { \ - uint8_t __b=(uint8_t)(s)[(i)++]; \ - if(U8_IS_LEAD(__b)) { \ - uint8_t __count=U8_COUNT_TRAIL_BYTES(__b); \ - if((i)+__count>(length) && (length)>=0) { \ - __count=(uint8_t)((length)-(i)); \ - } \ - while(__count>0 && U8_IS_TRAIL((s)[i])) { \ - ++(i); \ - --__count; \ + uint8_t __b=(s)[(i)++]; \ + if(U8_IS_LEAD(__b) && (i)!=(length)) { \ + uint8_t __t1=(s)[i]; \ + if((0xe0<=__b && __b<0xf0)) { \ + if(U8_IS_VALID_LEAD3_AND_T1(__b, __t1) && \ + ++(i)!=(length) && U8_IS_TRAIL((s)[i])) { \ + ++(i); \ + } \ + } else if(__b<0xe0) { \ + if(U8_IS_TRAIL(__t1)) { \ + ++(i); \ + } \ + } else /* c>=0xf0 */ { \ + if(U8_IS_VALID_LEAD4_AND_T1(__b, __t1) && \ + ++(i)!=(length) && U8_IS_TRAIL((s)[i]) && \ + ++(i)!=(length) && U8_IS_TRAIL((s)[i])) { \ + ++(i); \ + } \ } \ } \ } @@ -615,7 +643,7 @@ utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i); /* c is a trail byte */ \ (c)&=0x3f; \ for(;;) { \ - __b=(uint8_t)(s)[--(i)]; \ + __b=(s)[--(i)]; \ if(__b>=0xc0) { \ U8_MASK_LEAD_BYTE(__b, __count); \ (c)|=(UChar32)__b<<__shift; \ @@ -651,7 +679,7 @@ utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i); */ #define U8_PREV(s, start, i, c) { \ (c)=(uint8_t)(s)[--(i)]; \ - if((c)>=0x80) { \ + if(!U8_IS_SINGLE(c)) { \ (c)=utf8_prevCharSafeBody((const uint8_t *)s, start, &(i), c, -1); \ } \ } @@ -682,7 +710,7 @@ utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i); */ #define U8_PREV_OR_FFFD(s, start, i, c) { \ (c)=(uint8_t)(s)[--(i)]; \ - if((c)>=0x80) { \ + if(!U8_IS_SINGLE(c)) { \ (c)=utf8_prevCharSafeBody((const uint8_t *)s, start, &(i), c, -3); \ } \ } -- cgit v1.2.3