|
jpayne@69
|
1 // © 2016 and later: Unicode, Inc. and others.
|
|
jpayne@69
|
2 // License & terms of use: http://www.unicode.org/copyright.html
|
|
jpayne@69
|
3 /*
|
|
jpayne@69
|
4 *******************************************************************************
|
|
jpayne@69
|
5 *
|
|
jpayne@69
|
6 * Copyright (C) 1999-2011, International Business Machines
|
|
jpayne@69
|
7 * Corporation and others. All Rights Reserved.
|
|
jpayne@69
|
8 *
|
|
jpayne@69
|
9 *******************************************************************************
|
|
jpayne@69
|
10 * file name: utf.h
|
|
jpayne@69
|
11 * encoding: UTF-8
|
|
jpayne@69
|
12 * tab size: 8 (not used)
|
|
jpayne@69
|
13 * indentation:4
|
|
jpayne@69
|
14 *
|
|
jpayne@69
|
15 * created on: 1999sep09
|
|
jpayne@69
|
16 * created by: Markus W. Scherer
|
|
jpayne@69
|
17 */
|
|
jpayne@69
|
18
|
|
jpayne@69
|
19 /**
|
|
jpayne@69
|
20 * \file
|
|
jpayne@69
|
21 * \brief C API: Code point macros
|
|
jpayne@69
|
22 *
|
|
jpayne@69
|
23 * This file defines macros for checking whether a code point is
|
|
jpayne@69
|
24 * a surrogate or a non-character etc.
|
|
jpayne@69
|
25 *
|
|
jpayne@69
|
26 * If U_NO_DEFAULT_INCLUDE_UTF_HEADERS is 0 then utf.h is included by utypes.h
|
|
jpayne@69
|
27 * and itself includes utf8.h and utf16.h after some
|
|
jpayne@69
|
28 * common definitions.
|
|
jpayne@69
|
29 * If U_NO_DEFAULT_INCLUDE_UTF_HEADERS is 1 then each of these headers must be
|
|
jpayne@69
|
30 * included explicitly if their definitions are used.
|
|
jpayne@69
|
31 *
|
|
jpayne@69
|
32 * utf8.h and utf16.h define macros for efficiently getting code points
|
|
jpayne@69
|
33 * in and out of UTF-8/16 strings.
|
|
jpayne@69
|
34 * utf16.h macros have "U16_" prefixes.
|
|
jpayne@69
|
35 * utf8.h defines similar macros with "U8_" prefixes for UTF-8 string handling.
|
|
jpayne@69
|
36 *
|
|
jpayne@69
|
37 * ICU mostly processes 16-bit Unicode strings.
|
|
jpayne@69
|
38 * Most of the time, such strings are well-formed UTF-16.
|
|
jpayne@69
|
39 * Single, unpaired surrogates must be handled as well, and are treated in ICU
|
|
jpayne@69
|
40 * like regular code points where possible.
|
|
jpayne@69
|
41 * (Pairs of surrogate code points are indistinguishable from supplementary
|
|
jpayne@69
|
42 * code points encoded as pairs of supplementary code units.)
|
|
jpayne@69
|
43 *
|
|
jpayne@69
|
44 * In fact, almost all Unicode code points in normal text (>99%)
|
|
jpayne@69
|
45 * are on the BMP (<=U+ffff) and even <=U+d7ff.
|
|
jpayne@69
|
46 * ICU functions handle supplementary code points (U+10000..U+10ffff)
|
|
jpayne@69
|
47 * but are optimized for the much more frequently occurring BMP code points.
|
|
jpayne@69
|
48 *
|
|
jpayne@69
|
49 * umachine.h defines UChar to be an unsigned 16-bit integer.
|
|
jpayne@69
|
50 * Since ICU 59, ICU uses char16_t in C++, UChar only in C,
|
|
jpayne@69
|
51 * and defines UChar=char16_t by default. See the UChar API docs for details.
|
|
jpayne@69
|
52 *
|
|
jpayne@69
|
53 * UChar32 is defined to be a signed 32-bit integer (int32_t), large enough for a 21-bit
|
|
jpayne@69
|
54 * Unicode code point (Unicode scalar value, 0..0x10ffff) and U_SENTINEL (-1).
|
|
jpayne@69
|
55 * Before ICU 2.4, the definition of UChar32 was similarly platform-dependent as
|
|
jpayne@69
|
56 * the definition of UChar. For details see the documentation for UChar32 itself.
|
|
jpayne@69
|
57 *
|
|
jpayne@69
|
58 * utf.h defines a small number of C macros for single Unicode code points.
|
|
jpayne@69
|
59 * These are simple checks for surrogates and non-characters.
|
|
jpayne@69
|
60 * For actual Unicode character properties see uchar.h.
|
|
jpayne@69
|
61 *
|
|
jpayne@69
|
62 * By default, string operations must be done with error checking in case
|
|
jpayne@69
|
63 * a string is not well-formed UTF-16 or UTF-8.
|
|
jpayne@69
|
64 *
|
|
jpayne@69
|
65 * The U16_ macros detect if a surrogate code unit is unpaired
|
|
jpayne@69
|
66 * (lead unit without trail unit or vice versa) and just return the unit itself
|
|
jpayne@69
|
67 * as the code point.
|
|
jpayne@69
|
68 *
|
|
jpayne@69
|
69 * The U8_ macros detect illegal byte sequences and return a negative value.
|
|
jpayne@69
|
70 * Starting with ICU 60, the observable length of a single illegal byte sequence
|
|
jpayne@69
|
71 * skipped by one of these macros follows the Unicode 6+ recommendation
|
|
jpayne@69
|
72 * which is consistent with the W3C Encoding Standard.
|
|
jpayne@69
|
73 *
|
|
jpayne@69
|
74 * There are ..._OR_FFFD versions of both U16_ and U8_ macros
|
|
jpayne@69
|
75 * that return U+FFFD for illegal code unit sequences.
|
|
jpayne@69
|
76 *
|
|
jpayne@69
|
77 * The regular "safe" macros require that the initial, passed-in string index
|
|
jpayne@69
|
78 * is within bounds. They only check the index when they read more than one
|
|
jpayne@69
|
79 * code unit. This is usually done with code similar to the following loop:
|
|
jpayne@69
|
80 * <pre>while(i<length) {
|
|
jpayne@69
|
81 * U16_NEXT(s, i, length, c);
|
|
jpayne@69
|
82 * // use c
|
|
jpayne@69
|
83 * }</pre>
|
|
jpayne@69
|
84 *
|
|
jpayne@69
|
85 * When it is safe to assume that text is well-formed UTF-16
|
|
jpayne@69
|
86 * (does not contain single, unpaired surrogates), then one can use
|
|
jpayne@69
|
87 * U16_..._UNSAFE macros.
|
|
jpayne@69
|
88 * These do not check for proper code unit sequences or truncated text and may
|
|
jpayne@69
|
89 * yield wrong results or even cause a crash if they are used with "malformed"
|
|
jpayne@69
|
90 * text.
|
|
jpayne@69
|
91 * In practice, U16_..._UNSAFE macros will produce slightly less code but
|
|
jpayne@69
|
92 * should not be faster because the processing is only different when a
|
|
jpayne@69
|
93 * surrogate code unit is detected, which will be rare.
|
|
jpayne@69
|
94 *
|
|
jpayne@69
|
95 * Similarly for UTF-8, there are "safe" macros without a suffix,
|
|
jpayne@69
|
96 * and U8_..._UNSAFE versions.
|
|
jpayne@69
|
97 * The performance differences are much larger here because UTF-8 provides so
|
|
jpayne@69
|
98 * many opportunities for malformed sequences.
|
|
jpayne@69
|
99 * The unsafe UTF-8 macros are entirely implemented inside the macro definitions
|
|
jpayne@69
|
100 * and are fast, while the safe UTF-8 macros call functions for some complicated cases.
|
|
jpayne@69
|
101 *
|
|
jpayne@69
|
102 * Unlike with UTF-16, malformed sequences cannot be expressed with distinct
|
|
jpayne@69
|
103 * code point values (0..U+10ffff). They are indicated with negative values instead.
|
|
jpayne@69
|
104 *
|
|
jpayne@69
|
105 * For more information see the ICU User Guide Strings chapter
|
|
jpayne@69
|
106 * (http://userguide.icu-project.org/strings).
|
|
jpayne@69
|
107 *
|
|
jpayne@69
|
108 * <em>Usage:</em>
|
|
jpayne@69
|
109 * ICU coding guidelines for if() statements should be followed when using these macros.
|
|
jpayne@69
|
110 * Compound statements (curly braces {}) must be used for if-else-while...
|
|
jpayne@69
|
111 * bodies and all macro statements should be terminated with semicolon.
|
|
jpayne@69
|
112 *
|
|
jpayne@69
|
113 * @stable ICU 2.4
|
|
jpayne@69
|
114 */
|
|
jpayne@69
|
115
|
|
jpayne@69
|
116 #ifndef __UTF_H__
|
|
jpayne@69
|
117 #define __UTF_H__
|
|
jpayne@69
|
118
|
|
jpayne@69
|
119 #include "unicode/umachine.h"
|
|
jpayne@69
|
120 /* include the utfXX.h after the following definitions */
|
|
jpayne@69
|
121
|
|
jpayne@69
|
122 /* single-code point definitions -------------------------------------------- */
|
|
jpayne@69
|
123
|
|
jpayne@69
|
124 /**
|
|
jpayne@69
|
125 * Is this code point a Unicode noncharacter?
|
|
jpayne@69
|
126 * @param c 32-bit code point
|
|
jpayne@69
|
127 * @return TRUE or FALSE
|
|
jpayne@69
|
128 * @stable ICU 2.4
|
|
jpayne@69
|
129 */
|
|
jpayne@69
|
130 #define U_IS_UNICODE_NONCHAR(c) \
|
|
jpayne@69
|
131 ((c)>=0xfdd0 && \
|
|
jpayne@69
|
132 ((c)<=0xfdef || ((c)&0xfffe)==0xfffe) && (c)<=0x10ffff)
|
|
jpayne@69
|
133
|
|
jpayne@69
|
134 /**
|
|
jpayne@69
|
135 * Is c a Unicode code point value (0..U+10ffff)
|
|
jpayne@69
|
136 * that can be assigned a character?
|
|
jpayne@69
|
137 *
|
|
jpayne@69
|
138 * Code points that are not characters include:
|
|
jpayne@69
|
139 * - single surrogate code points (U+d800..U+dfff, 2048 code points)
|
|
jpayne@69
|
140 * - the last two code points on each plane (U+__fffe and U+__ffff, 34 code points)
|
|
jpayne@69
|
141 * - U+fdd0..U+fdef (new with Unicode 3.1, 32 code points)
|
|
jpayne@69
|
142 * - the highest Unicode code point value is U+10ffff
|
|
jpayne@69
|
143 *
|
|
jpayne@69
|
144 * This means that all code points below U+d800 are character code points,
|
|
jpayne@69
|
145 * and that boundary is tested first for performance.
|
|
jpayne@69
|
146 *
|
|
jpayne@69
|
147 * @param c 32-bit code point
|
|
jpayne@69
|
148 * @return TRUE or FALSE
|
|
jpayne@69
|
149 * @stable ICU 2.4
|
|
jpayne@69
|
150 */
|
|
jpayne@69
|
151 #define U_IS_UNICODE_CHAR(c) \
|
|
jpayne@69
|
152 ((uint32_t)(c)<0xd800 || \
|
|
jpayne@69
|
153 (0xdfff<(c) && (c)<=0x10ffff && !U_IS_UNICODE_NONCHAR(c)))
|
|
jpayne@69
|
154
|
|
jpayne@69
|
155 /**
|
|
jpayne@69
|
156 * Is this code point a BMP code point (U+0000..U+ffff)?
|
|
jpayne@69
|
157 * @param c 32-bit code point
|
|
jpayne@69
|
158 * @return TRUE or FALSE
|
|
jpayne@69
|
159 * @stable ICU 2.8
|
|
jpayne@69
|
160 */
|
|
jpayne@69
|
161 #define U_IS_BMP(c) ((uint32_t)(c)<=0xffff)
|
|
jpayne@69
|
162
|
|
jpayne@69
|
163 /**
|
|
jpayne@69
|
164 * Is this code point a supplementary code point (U+10000..U+10ffff)?
|
|
jpayne@69
|
165 * @param c 32-bit code point
|
|
jpayne@69
|
166 * @return TRUE or FALSE
|
|
jpayne@69
|
167 * @stable ICU 2.8
|
|
jpayne@69
|
168 */
|
|
jpayne@69
|
169 #define U_IS_SUPPLEMENTARY(c) ((uint32_t)((c)-0x10000)<=0xfffff)
|
|
jpayne@69
|
170
|
|
jpayne@69
|
171 /**
|
|
jpayne@69
|
172 * Is this code point a lead surrogate (U+d800..U+dbff)?
|
|
jpayne@69
|
173 * @param c 32-bit code point
|
|
jpayne@69
|
174 * @return TRUE or FALSE
|
|
jpayne@69
|
175 * @stable ICU 2.4
|
|
jpayne@69
|
176 */
|
|
jpayne@69
|
177 #define U_IS_LEAD(c) (((c)&0xfffffc00)==0xd800)
|
|
jpayne@69
|
178
|
|
jpayne@69
|
179 /**
|
|
jpayne@69
|
180 * Is this code point a trail surrogate (U+dc00..U+dfff)?
|
|
jpayne@69
|
181 * @param c 32-bit code point
|
|
jpayne@69
|
182 * @return TRUE or FALSE
|
|
jpayne@69
|
183 * @stable ICU 2.4
|
|
jpayne@69
|
184 */
|
|
jpayne@69
|
185 #define U_IS_TRAIL(c) (((c)&0xfffffc00)==0xdc00)
|
|
jpayne@69
|
186
|
|
jpayne@69
|
187 /**
|
|
jpayne@69
|
188 * Is this code point a surrogate (U+d800..U+dfff)?
|
|
jpayne@69
|
189 * @param c 32-bit code point
|
|
jpayne@69
|
190 * @return TRUE or FALSE
|
|
jpayne@69
|
191 * @stable ICU 2.4
|
|
jpayne@69
|
192 */
|
|
jpayne@69
|
193 #define U_IS_SURROGATE(c) (((c)&0xfffff800)==0xd800)
|
|
jpayne@69
|
194
|
|
jpayne@69
|
195 /**
|
|
jpayne@69
|
196 * Assuming c is a surrogate code point (U_IS_SURROGATE(c)),
|
|
jpayne@69
|
197 * is it a lead surrogate?
|
|
jpayne@69
|
198 * @param c 32-bit code point
|
|
jpayne@69
|
199 * @return TRUE or FALSE
|
|
jpayne@69
|
200 * @stable ICU 2.4
|
|
jpayne@69
|
201 */
|
|
jpayne@69
|
202 #define U_IS_SURROGATE_LEAD(c) (((c)&0x400)==0)
|
|
jpayne@69
|
203
|
|
jpayne@69
|
204 /**
|
|
jpayne@69
|
205 * Assuming c is a surrogate code point (U_IS_SURROGATE(c)),
|
|
jpayne@69
|
206 * is it a trail surrogate?
|
|
jpayne@69
|
207 * @param c 32-bit code point
|
|
jpayne@69
|
208 * @return TRUE or FALSE
|
|
jpayne@69
|
209 * @stable ICU 4.2
|
|
jpayne@69
|
210 */
|
|
jpayne@69
|
211 #define U_IS_SURROGATE_TRAIL(c) (((c)&0x400)!=0)
|
|
jpayne@69
|
212
|
|
jpayne@69
|
213 /* include the utfXX.h ------------------------------------------------------ */
|
|
jpayne@69
|
214
|
|
jpayne@69
|
215 #if !U_NO_DEFAULT_INCLUDE_UTF_HEADERS
|
|
jpayne@69
|
216
|
|
jpayne@69
|
217 #include "unicode/utf8.h"
|
|
jpayne@69
|
218 #include "unicode/utf16.h"
|
|
jpayne@69
|
219
|
|
jpayne@69
|
220 /* utf_old.h contains deprecated, pre-ICU 2.4 definitions */
|
|
jpayne@69
|
221 #include "unicode/utf_old.h"
|
|
jpayne@69
|
222
|
|
jpayne@69
|
223 #endif /* !U_NO_DEFAULT_INCLUDE_UTF_HEADERS */
|
|
jpayne@69
|
224
|
|
jpayne@69
|
225 #endif /* __UTF_H__ */
|
