Movatterモバイル変換

Go to the documentation of this file.

2 // License & terms of use: http://www.unicode.org/copyright.html

3 /*

4 *******************************************************************************

5 *

8 *

9 *******************************************************************************

10 * file name: utf8.h

11 * encoding: UTF-8

12 * tab size: 8 (not used)

13 * indentation:4

14 *

15 * created on: 1999sep13

16 * created by: Markus W. Scherer

17 */

34 #ifndef __UTF8_H__

35 #define __UTF8_H__

37 #include <stdbool.h>

38 #include "unicode/umachine.h"

39 #ifndef __UTF_H__

40 # include "unicode/utf.h"

41 #endif

43 /* internal definitions ----------------------------------------------------- */

56 #define U8_COUNT_TRAIL_BYTES(leadByte) \

57 (U8_IS_LEAD(leadByte) ? \

58 ((uint8_t)(leadByte)>=0xe0)+((uint8_t)(leadByte)>=0xf0)+1 : 0)

71 #define U8_COUNT_TRAIL_BYTES_UNSAFE(leadByte) \

72 (((uint8_t)(leadByte)>=0xc2)+((uint8_t)(leadByte)>=0xe0)+((uint8_t)(leadByte)>=0xf0))

81 #define U8_MASK_LEAD_BYTE(leadByte, countTrailBytes) ((leadByte)&=(1<<(6-(countTrailBytes)))-1)

91 #define U8_LEAD3_T1_BITS "\x20\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x10\x30\x30"

98 #define U8_IS_VALID_LEAD3_AND_T1(lead, t1) (U8_LEAD3_T1_BITS[(lead)&0xf]&(1<<((uint8_t)(t1)>>5)))

108 #define U8_LEAD4_T1_BITS "\x00\x00\x00\x00\x00\x00\x00\x00\x1E\x0F\x0F\x0F\x00\x00\x00\x00"

109

115 #define U8_IS_VALID_LEAD4_AND_T1(lead, t1) (U8_LEAD4_T1_BITS[(uint8_t)(t1)>>4]&(1<<((lead)&7)))

116

126 U_CAPI UChar32 U_EXPORT2

127 utf8_nextCharSafeBody(const uint8_t *s, int32_t *pi, int32_t length,UChar32 c, int8_t strict);

128

138 U_CAPI int32_t U_EXPORT2

139 utf8_appendCharSafeBody(uint8_t *s, int32_t i, int32_t length,UChar32 c,UBool *pIsError);

140

150 U_CAPI UChar32 U_EXPORT2

151 utf8_prevCharSafeBody(const uint8_t *s, int32_t start, int32_t *pi,UChar32 c, int8_t strict);

152

162 U_CAPI int32_t U_EXPORT2

163 utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i);

164

165 /* single-code point definitions -------------------------------------------- */

166

173 #define U8_IS_SINGLE(c) ((int8_t)(c)>=0)

174

181 #define U8_IS_LEAD(c) ((uint8_t)((c)-0xc2)<=0x32)

182 // 0x32=0xf4-0xc2

183

190 #define U8_IS_TRAIL(c) ((int8_t)(c)<-0x40)

191

199 #define U8_LENGTH(c) \

200 ((uint32_t)(c)<=0x7f ? 1 : \

201 ((uint32_t)(c)<=0x7ff ? 2 : \

202 ((uint32_t)(c)<=0xd7ff ? 3 : \

203 ((uint32_t)(c)<=0xdfff || (uint32_t)(c)>0x10ffff ? 0 : \

204 ((uint32_t)(c)<=0xffff ? 3 : 4)\

205 ) \

206 ) \

207 ) \

208 )

209

215 #define U8_MAX_LENGTH 4

216

217 #ifndef U_HIDE_DRAFT_API

218

228 #define U8_LENGTH_FROM_LEAD_BYTE(leadByte) (U8_COUNT_TRAIL_BYTES(leadByte) + 1)

229

239 #define U8_LENGTH_FROM_LEAD_BYTE_UNSAFE(leadByte) (U8_COUNT_TRAIL_BYTES_UNSAFE(leadByte) + 1)

240

241 #endif// U_HIDE_DRAFT_API

242

259 #define U8_GET_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \

260 int32_t _u8_get_unsafe_index=(int32_t)(i); \

261 U8_SET_CP_START_UNSAFE(s, _u8_get_unsafe_index); \

262 U8_NEXT_UNSAFE(s, _u8_get_unsafe_index, c); \

263 } UPRV_BLOCK_MACRO_END

264

286 #define U8_GET(s, start, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \

287 int32_t _u8_get_index=(i); \

288 U8_SET_CP_START(s, start, _u8_get_index); \

289 U8_NEXT(s, _u8_get_index, length, c); \

290 } UPRV_BLOCK_MACRO_END

291

317 #define U8_GET_OR_FFFD(s, start, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \

318 int32_t _u8_get_index=(i); \

319 U8_SET_CP_START(s, start, _u8_get_index); \

320 U8_NEXT_OR_FFFD(s, _u8_get_index, length, c); \

321 } UPRV_BLOCK_MACRO_END

322

323 /* definitions with forward iteration --------------------------------------- */

324

342 #define U8_NEXT_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \

343 (c)=(uint8_t)(s)[(i)++]; \

344 if(!U8_IS_SINGLE(c)) { \

345 if((c)<0xe0) { \

346 (c)=(((c)&0x1f)<<6)|((s)[(i)++]&0x3f); \

347 } else if((c)<0xf0) { \

348 /* no need for (c&0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ \

349 (c)=(UChar)(((c)<<12)|(((s)[i]&0x3f)<<6)|((s)[(i)+1]&0x3f)); \

350 (i)+=2; \

351 } else { \

352 (c)=(((c)&7)<<18)|(((s)[i]&0x3f)<<12)|(((s)[(i)+1]&0x3f)<<6)|((s)[(i)+2]&0x3f); \

353 (i)+=3; \

354 } \

355 } \

356 } UPRV_BLOCK_MACRO_END

357

378 #define U8_NEXT(s, i, length, c) U8_INTERNAL_NEXT_OR_SUB(s, i, length, c, U_SENTINEL)

379

404 #define U8_NEXT_OR_FFFD(s, i, length, c) U8_INTERNAL_NEXT_OR_SUB(s, i, length, c, 0xfffd)

405

407 #define U8_INTERNAL_NEXT_OR_SUB(s, i, length, c, sub) UPRV_BLOCK_MACRO_BEGIN { \

408 (c)=(uint8_t)(s)[(i)++]; \

409 if(!U8_IS_SINGLE(c)) { \

410 uint8_t __t = 0; \

411 if((i)!=(length) && \

412 /* fetch/validate/assemble all but last trail byte */ \

413 ((c)>=0xe0 ? \

414 ((c)<0xf0 ?/* U+0800..U+FFFF except surrogates */ \

415 U8_LEAD3_T1_BITS[(c)&=0xf]&(1<<((__t=(s)[i])>>5)) && \

416 (__t&=0x3f, 1) \

417 :/* U+10000..U+10FFFF */ \

418 ((c)-=0xf0)<=4 && \

419 U8_LEAD4_T1_BITS[(__t=(s)[i])>>4]&(1<<(c)) && \

420 ((c)=((c)<<6)|(__t&0x3f), ++(i)!=(length)) && \

421 (__t=(s)[i]-0x80)<=0x3f) && \

422 /* valid second-to-last trail byte */ \

423 ((c)=((c)<<6)|__t, ++(i)!=(length)) \

424 :/* U+0080..U+07FF */ \

425 (c)>=0xc2 && ((c)&=0x1f, 1)) && \

426 /* last trail byte */ \

427 (__t=(s)[i]-0x80)<=0x3f && \

428 ((c)=((c)<<6)|__t, ++(i), 1)) { \

429 } else { \

430 (c)=(sub);/* ill-formed*/ \

431 } \

432 } \

433 } UPRV_BLOCK_MACRO_END

434

448 #define U8_APPEND_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \

449 uint32_t __uc=(c); \

450 if(__uc<=0x7f) { \

451 (s)[(i)++]=(uint8_t)__uc; \

452 } else { \

453 if(__uc<=0x7ff) { \

454 (s)[(i)++]=(uint8_t)((__uc>>6)|0xc0); \

455 } else { \

456 if(__uc<=0xffff) { \

457 (s)[(i)++]=(uint8_t)((__uc>>12)|0xe0); \

458 } else { \

459 (s)[(i)++]=(uint8_t)((__uc>>18)|0xf0); \

460 (s)[(i)++]=(uint8_t)(((__uc>>12)&0x3f)|0x80); \

461 } \

462 (s)[(i)++]=(uint8_t)(((__uc>>6)&0x3f)|0x80); \

463 } \

464 (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \

465 } \

466 } UPRV_BLOCK_MACRO_END

467

485 #define U8_APPEND(s, i, capacity, c, isError) UPRV_BLOCK_MACRO_BEGIN { \

486 uint32_t __uc=(c); \

487 if(__uc<=0x7f) { \

488 (s)[(i)++]=(uint8_t)__uc; \

489 } else if(__uc<=0x7ff && (i)+1<(capacity)) { \

490 (s)[(i)++]=(uint8_t)((__uc>>6)|0xc0); \

491 (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \

492 } else if((__uc<=0xd7ff || (0xe000<=__uc && __uc<=0xffff)) && (i)+2<(capacity)) { \

493 (s)[(i)++]=(uint8_t)((__uc>>12)|0xe0); \

494 (s)[(i)++]=(uint8_t)(((__uc>>6)&0x3f)|0x80); \

495 (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \

496 } else if(0xffff<__uc && __uc<=0x10ffff && (i)+3<(capacity)) { \

497 (s)[(i)++]=(uint8_t)((__uc>>18)|0xf0); \

498 (s)[(i)++]=(uint8_t)(((__uc>>12)&0x3f)|0x80); \

499 (s)[(i)++]=(uint8_t)(((__uc>>6)&0x3f)|0x80); \

500 (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \

501 } else { \

502 (isError)=true; \

503 } \

504 } UPRV_BLOCK_MACRO_END

505

516 #define U8_FWD_1_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \

517 (i)+=1+U8_COUNT_TRAIL_BYTES_UNSAFE((s)[i]); \

518 } UPRV_BLOCK_MACRO_END

519

533 #define U8_FWD_1(s, i, length) UPRV_BLOCK_MACRO_BEGIN { \

534 uint8_t __b=(s)[(i)++]; \

535 if(U8_IS_LEAD(__b) && (i)!=(length)) { \

536 uint8_t __t1=(s)[i]; \

537 if((0xe0<=__b && __b<0xf0)) { \

538 if(U8_IS_VALID_LEAD3_AND_T1(__b, __t1) && \

539 ++(i)!=(length) && U8_IS_TRAIL((s)[i])) { \

540 ++(i); \

541 } \

542 } else if(__b<0xe0) { \

543 if(U8_IS_TRAIL(__t1)) { \

544 ++(i); \

545 } \

546 } else/* b>=0xf0 */ { \

547 if(U8_IS_VALID_LEAD4_AND_T1(__b, __t1) && \

548 ++(i)!=(length) && U8_IS_TRAIL((s)[i]) && \

549 ++(i)!=(length) && U8_IS_TRAIL((s)[i])) { \

550 ++(i); \

551 } \

552 } \

553 } \

554 } UPRV_BLOCK_MACRO_END

555

568 #define U8_FWD_N_UNSAFE(s, i, n) UPRV_BLOCK_MACRO_BEGIN { \

569 int32_t __N=(n); \

570 while(__N>0) { \

571 U8_FWD_1_UNSAFE(s, i); \

572 --__N; \

573 } \

574 } UPRV_BLOCK_MACRO_END

575

591 #define U8_FWD_N(s, i, length, n) UPRV_BLOCK_MACRO_BEGIN { \

592 int32_t __N=(n); \

593 while(__N>0 && ((i)<(length) || ((length)<0 && (s)[i]!=0))) { \

594 U8_FWD_1(s, i, length); \

595 --__N; \

596 } \

597 } UPRV_BLOCK_MACRO_END

598

612 #define U8_SET_CP_START_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \

613 while(U8_IS_TRAIL((s)[i])) { --(i); } \

614 } UPRV_BLOCK_MACRO_END

615

633 #define U8_SET_CP_START(s, start, i) UPRV_BLOCK_MACRO_BEGIN { \

634 if(U8_IS_TRAIL((s)[(i)])) { \

635 (i)=utf8_back1SafeBody(s, start, (i)); \

636 } \

637 } UPRV_BLOCK_MACRO_END

638

665 #define U8_TRUNCATE_IF_INCOMPLETE(s, start, length) UPRV_BLOCK_MACRO_BEGIN { \

666 if((length)>(start)) { \

667 uint8_t __b1=s[(length)-1]; \

668 if(U8_IS_SINGLE(__b1)) { \

669 /* common ASCII character */ \

670 } else if(U8_IS_LEAD(__b1)) { \

671 --(length); \

672 } else if(U8_IS_TRAIL(__b1) && ((length)-2)>=(start)) { \

673 uint8_t __b2=s[(length)-2]; \

674 if(0xe0<=__b2 && __b2<=0xf4) { \

675 if(__b2<0xf0 ? U8_IS_VALID_LEAD3_AND_T1(__b2, __b1) : \

676 U8_IS_VALID_LEAD4_AND_T1(__b2, __b1)) { \

677 (length)-=2; \

678 } \

679 } else if(U8_IS_TRAIL(__b2) && ((length)-3)>=(start)) { \

680 uint8_t __b3=s[(length)-3]; \

681 if(0xf0<=__b3 && __b3<=0xf4 && U8_IS_VALID_LEAD4_AND_T1(__b3, __b2)) { \

682 (length)-=3; \

683 } \

684 } \

685 } \

686 } \

687 } UPRV_BLOCK_MACRO_END

688

689 /* definitions with backward iteration -------------------------------------- */

690

710 #define U8_PREV_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \

711 (c)=(uint8_t)(s)[--(i)]; \

712 if(!U8_IS_SINGLE(c)) { \

713 uint8_t __b, __count=1, __shift=6; \

714 \

715 /* c is a trail byte */ \

716 (c)&=0x3f; \

717 for(;;) { \

718 __b=(s)[--(i)]; \

719 if(__b>=0xc0) { \

720 U8_MASK_LEAD_BYTE(__b, __count); \

721 (c)|=(UChar32)__b<<__shift; \

722 break; \

723 } else { \

724 (c)|=(UChar32)(__b&0x3f)<<__shift; \

725 ++__count; \

726 __shift+=6; \

727 } \

728 } \

729 } \

730 } UPRV_BLOCK_MACRO_END

731

752 #define U8_PREV(s, start, i, c) UPRV_BLOCK_MACRO_BEGIN { \

753 (c)=(uint8_t)(s)[--(i)]; \

754 if(!U8_IS_SINGLE(c)) { \

755 (c)=utf8_prevCharSafeBody((const uint8_t *)s, start, &(i), c, -1); \

756 } \

757 } UPRV_BLOCK_MACRO_END

758

783 #define U8_PREV_OR_FFFD(s, start, i, c) UPRV_BLOCK_MACRO_BEGIN { \

784 (c)=(uint8_t)(s)[--(i)]; \

785 if(!U8_IS_SINGLE(c)) { \

786 (c)=utf8_prevCharSafeBody((const uint8_t *)s, start, &(i), c, -3); \

787 } \

788 } UPRV_BLOCK_MACRO_END

789

801 #define U8_BACK_1_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \

802 while(U8_IS_TRAIL((s)[--(i)])) {} \

803 } UPRV_BLOCK_MACRO_END

804

817 #define U8_BACK_1(s, start, i) UPRV_BLOCK_MACRO_BEGIN { \

818 if(U8_IS_TRAIL((s)[--(i)])) { \

819 (i)=utf8_back1SafeBody(s, start, (i)); \

820 } \

821 } UPRV_BLOCK_MACRO_END

822

836 #define U8_BACK_N_UNSAFE(s, i, n) UPRV_BLOCK_MACRO_BEGIN { \

837 int32_t __N=(n); \

838 while(__N>0) { \

839 U8_BACK_1_UNSAFE(s, i); \

840 --__N; \

841 } \

842 } UPRV_BLOCK_MACRO_END

843

858 #define U8_BACK_N(s, start, i, n) UPRV_BLOCK_MACRO_BEGIN { \

859 int32_t __N=(n); \

860 while(__N>0 && (i)>(start)) { \

861 U8_BACK_1(s, start, i); \

862 --__N; \

863 } \

864 } UPRV_BLOCK_MACRO_END

865

879 #define U8_SET_CP_LIMIT_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \

880 U8_BACK_1_UNSAFE(s, i); \

881 U8_FWD_1_UNSAFE(s, i); \

882 } UPRV_BLOCK_MACRO_END

883

901 #define U8_SET_CP_LIMIT(s, start, i, length) UPRV_BLOCK_MACRO_BEGIN { \

902 if((start)<(i) && ((i)<(length) || (length)<0)) { \

903 U8_BACK_1(s, start, i); \

904 U8_FWD_1(s, i, length); \

905 } \

906 } UPRV_BLOCK_MACRO_END

907

908 #endif

umachine.h

Basic types and constants for UTF.

UChar32

int32_t UChar32

Define UChar32 as a type for single Unicode code points.

Definition:umachine.h:449

UBool

int8_t UBool

The ICU boolean type, a signed-byte integer.

Definition:umachine.h:269

U_CAPI

#define U_CAPI

This is used to declare a function as a public ICU C API.

Definition:umachine.h:110

utf8_appendCharSafeBody

U_CAPI int32_t utf8_appendCharSafeBody(uint8_t *s, int32_t i, int32_t length, UChar32 c, UBool *pIsError)

Function for handling "append code point" with error-checking.

utf8_back1SafeBody

U_CAPI int32_t utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i)

Function for handling "skip backward one code point" with error-checking.

utf8_nextCharSafeBody

U_CAPI UChar32 utf8_nextCharSafeBody(const uint8_t *s, int32_t *pi, int32_t length, UChar32 c, int8_t strict)

Function for handling "next code point" with error-checking.

utf8_prevCharSafeBody

U_CAPI UChar32 utf8_prevCharSafeBody(const uint8_t *s, int32_t start, int32_t *pi, UChar32 c, int8_t strict)

Function for handling "previous code point" with error-checking.

utf.h

C API: Code point macros.