|
jpayne@69
|
1 // © 2016 and later: Unicode, Inc. and others.
|
|
jpayne@69
|
2 // License & terms of use: http://www.unicode.org/copyright.html
|
|
jpayne@69
|
3 /*
|
|
jpayne@69
|
4 *******************************************************************************
|
|
jpayne@69
|
5 * Copyright (c) 1996-2016, International Business Machines Corporation
|
|
jpayne@69
|
6 * and others. All Rights Reserved.
|
|
jpayne@69
|
7 *******************************************************************************
|
|
jpayne@69
|
8 * File unorm.h
|
|
jpayne@69
|
9 *
|
|
jpayne@69
|
10 * Created by: Vladimir Weinstein 12052000
|
|
jpayne@69
|
11 *
|
|
jpayne@69
|
12 * Modification history :
|
|
jpayne@69
|
13 *
|
|
jpayne@69
|
14 * Date Name Description
|
|
jpayne@69
|
15 * 02/01/01 synwee Added normalization quickcheck enum and method.
|
|
jpayne@69
|
16 */
|
|
jpayne@69
|
17 #ifndef UNORM_H
|
|
jpayne@69
|
18 #define UNORM_H
|
|
jpayne@69
|
19
|
|
jpayne@69
|
20 #include "unicode/utypes.h"
|
|
jpayne@69
|
21
|
|
jpayne@69
|
22 #if !UCONFIG_NO_NORMALIZATION
|
|
jpayne@69
|
23
|
|
jpayne@69
|
24 #include "unicode/uiter.h"
|
|
jpayne@69
|
25 #include "unicode/unorm2.h"
|
|
jpayne@69
|
26
|
|
jpayne@69
|
27 /**
|
|
jpayne@69
|
28 * \file
|
|
jpayne@69
|
29 * \brief C API: Unicode Normalization
|
|
jpayne@69
|
30 *
|
|
jpayne@69
|
31 * Old Unicode normalization API.
|
|
jpayne@69
|
32 *
|
|
jpayne@69
|
33 * This API has been replaced by the unorm2.h API and is only available
|
|
jpayne@69
|
34 * for backward compatibility. The functions here simply delegate to the
|
|
jpayne@69
|
35 * unorm2.h functions, for example unorm2_getInstance() and unorm2_normalize().
|
|
jpayne@69
|
36 * There is one exception: The new API does not provide a replacement for unorm_compare().
|
|
jpayne@69
|
37 * Its declaration has been moved to unorm2.h.
|
|
jpayne@69
|
38 *
|
|
jpayne@69
|
39 * <code>unorm_normalize</code> transforms Unicode text into an equivalent composed or
|
|
jpayne@69
|
40 * decomposed form, allowing for easier sorting and searching of text.
|
|
jpayne@69
|
41 * <code>unorm_normalize</code> supports the standard normalization forms described in
|
|
jpayne@69
|
42 * <a href="http://www.unicode.org/unicode/reports/tr15/" target="unicode">
|
|
jpayne@69
|
43 * Unicode Standard Annex #15: Unicode Normalization Forms</a>.
|
|
jpayne@69
|
44 *
|
|
jpayne@69
|
45 * Characters with accents or other adornments can be encoded in
|
|
jpayne@69
|
46 * several different ways in Unicode. For example, take the character A-acute.
|
|
jpayne@69
|
47 * In Unicode, this can be encoded as a single character (the
|
|
jpayne@69
|
48 * "composed" form):
|
|
jpayne@69
|
49 *
|
|
jpayne@69
|
50 * \code
|
|
jpayne@69
|
51 * 00C1 LATIN CAPITAL LETTER A WITH ACUTE
|
|
jpayne@69
|
52 * \endcode
|
|
jpayne@69
|
53 *
|
|
jpayne@69
|
54 * or as two separate characters (the "decomposed" form):
|
|
jpayne@69
|
55 *
|
|
jpayne@69
|
56 * \code
|
|
jpayne@69
|
57 * 0041 LATIN CAPITAL LETTER A
|
|
jpayne@69
|
58 * 0301 COMBINING ACUTE ACCENT
|
|
jpayne@69
|
59 * \endcode
|
|
jpayne@69
|
60 *
|
|
jpayne@69
|
61 * To a user of your program, however, both of these sequences should be
|
|
jpayne@69
|
62 * treated as the same "user-level" character "A with acute accent". When you are searching or
|
|
jpayne@69
|
63 * comparing text, you must ensure that these two sequences are treated
|
|
jpayne@69
|
64 * equivalently. In addition, you must handle characters with more than one
|
|
jpayne@69
|
65 * accent. Sometimes the order of a character's combining accents is
|
|
jpayne@69
|
66 * significant, while in other cases accent sequences in different orders are
|
|
jpayne@69
|
67 * really equivalent.
|
|
jpayne@69
|
68 *
|
|
jpayne@69
|
69 * Similarly, the string "ffi" can be encoded as three separate letters:
|
|
jpayne@69
|
70 *
|
|
jpayne@69
|
71 * \code
|
|
jpayne@69
|
72 * 0066 LATIN SMALL LETTER F
|
|
jpayne@69
|
73 * 0066 LATIN SMALL LETTER F
|
|
jpayne@69
|
74 * 0069 LATIN SMALL LETTER I
|
|
jpayne@69
|
75 * \endcode
|
|
jpayne@69
|
76 *
|
|
jpayne@69
|
77 * or as the single character
|
|
jpayne@69
|
78 *
|
|
jpayne@69
|
79 * \code
|
|
jpayne@69
|
80 * FB03 LATIN SMALL LIGATURE FFI
|
|
jpayne@69
|
81 * \endcode
|
|
jpayne@69
|
82 *
|
|
jpayne@69
|
83 * The ffi ligature is not a distinct semantic character, and strictly speaking
|
|
jpayne@69
|
84 * it shouldn't be in Unicode at all, but it was included for compatibility
|
|
jpayne@69
|
85 * with existing character sets that already provided it. The Unicode standard
|
|
jpayne@69
|
86 * identifies such characters by giving them "compatibility" decompositions
|
|
jpayne@69
|
87 * into the corresponding semantic characters. When sorting and searching, you
|
|
jpayne@69
|
88 * will often want to use these mappings.
|
|
jpayne@69
|
89 *
|
|
jpayne@69
|
90 * <code>unorm_normalize</code> helps solve these problems by transforming text into the
|
|
jpayne@69
|
91 * canonical composed and decomposed forms as shown in the first example above.
|
|
jpayne@69
|
92 * In addition, you can have it perform compatibility decompositions so that
|
|
jpayne@69
|
93 * you can treat compatibility characters the same as their equivalents.
|
|
jpayne@69
|
94 * Finally, <code>unorm_normalize</code> rearranges accents into the proper canonical
|
|
jpayne@69
|
95 * order, so that you do not have to worry about accent rearrangement on your
|
|
jpayne@69
|
96 * own.
|
|
jpayne@69
|
97 *
|
|
jpayne@69
|
98 * Form FCD, "Fast C or D", is also designed for collation.
|
|
jpayne@69
|
99 * It allows to work on strings that are not necessarily normalized
|
|
jpayne@69
|
100 * with an algorithm (like in collation) that works under "canonical closure", i.e., it treats precomposed
|
|
jpayne@69
|
101 * characters and their decomposed equivalents the same.
|
|
jpayne@69
|
102 *
|
|
jpayne@69
|
103 * It is not a normalization form because it does not provide for uniqueness of representation. Multiple strings
|
|
jpayne@69
|
104 * may be canonically equivalent (their NFDs are identical) and may all conform to FCD without being identical
|
|
jpayne@69
|
105 * themselves.
|
|
jpayne@69
|
106 *
|
|
jpayne@69
|
107 * The form is defined such that the "raw decomposition", the recursive canonical decomposition of each character,
|
|
jpayne@69
|
108 * results in a string that is canonically ordered. This means that precomposed characters are allowed for as long
|
|
jpayne@69
|
109 * as their decompositions do not need canonical reordering.
|
|
jpayne@69
|
110 *
|
|
jpayne@69
|
111 * Its advantage for a process like collation is that all NFD and most NFC texts - and many unnormalized texts -
|
|
jpayne@69
|
112 * already conform to FCD and do not need to be normalized (NFD) for such a process. The FCD quick check will
|
|
jpayne@69
|
113 * return UNORM_YES for most strings in practice.
|
|
jpayne@69
|
114 *
|
|
jpayne@69
|
115 * unorm_normalize(UNORM_FCD) may be implemented with UNORM_NFD.
|
|
jpayne@69
|
116 *
|
|
jpayne@69
|
117 * For more details on FCD see the collation design document:
|
|
jpayne@69
|
118 * http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm
|
|
jpayne@69
|
119 *
|
|
jpayne@69
|
120 * ICU collation performs either NFD or FCD normalization automatically if normalization
|
|
jpayne@69
|
121 * is turned on for the collator object.
|
|
jpayne@69
|
122 * Beyond collation and string search, normalized strings may be useful for string equivalence comparisons,
|
|
jpayne@69
|
123 * transliteration/transcription, unique representations, etc.
|
|
jpayne@69
|
124 *
|
|
jpayne@69
|
125 * The W3C generally recommends to exchange texts in NFC.
|
|
jpayne@69
|
126 * Note also that most legacy character encodings use only precomposed forms and often do not
|
|
jpayne@69
|
127 * encode any combining marks by themselves. For conversion to such character encodings the
|
|
jpayne@69
|
128 * Unicode text needs to be normalized to NFC.
|
|
jpayne@69
|
129 * For more usage examples, see the Unicode Standard Annex.
|
|
jpayne@69
|
130 */
|
|
jpayne@69
|
131
|
|
jpayne@69
|
132 // Do not conditionalize the following enum with #ifndef U_HIDE_DEPRECATED_API,
|
|
jpayne@69
|
133 // it is needed for layout of Normalizer object.
|
|
jpayne@69
|
134 #ifndef U_FORCE_HIDE_DEPRECATED_API
|
|
jpayne@69
|
135
|
|
jpayne@69
|
136 /**
|
|
jpayne@69
|
137 * Constants for normalization modes.
|
|
jpayne@69
|
138 * @deprecated ICU 56 Use unorm2.h instead.
|
|
jpayne@69
|
139 */
|
|
jpayne@69
|
140 typedef enum {
|
|
jpayne@69
|
141 /** No decomposition/composition. @deprecated ICU 56 Use unorm2.h instead. */
|
|
jpayne@69
|
142 UNORM_NONE = 1,
|
|
jpayne@69
|
143 /** Canonical decomposition. @deprecated ICU 56 Use unorm2.h instead. */
|
|
jpayne@69
|
144 UNORM_NFD = 2,
|
|
jpayne@69
|
145 /** Compatibility decomposition. @deprecated ICU 56 Use unorm2.h instead. */
|
|
jpayne@69
|
146 UNORM_NFKD = 3,
|
|
jpayne@69
|
147 /** Canonical decomposition followed by canonical composition. @deprecated ICU 56 Use unorm2.h instead. */
|
|
jpayne@69
|
148 UNORM_NFC = 4,
|
|
jpayne@69
|
149 /** Default normalization. @deprecated ICU 56 Use unorm2.h instead. */
|
|
jpayne@69
|
150 UNORM_DEFAULT = UNORM_NFC,
|
|
jpayne@69
|
151 /** Compatibility decomposition followed by canonical composition. @deprecated ICU 56 Use unorm2.h instead. */
|
|
jpayne@69
|
152 UNORM_NFKC =5,
|
|
jpayne@69
|
153 /** "Fast C or D" form. @deprecated ICU 56 Use unorm2.h instead. */
|
|
jpayne@69
|
154 UNORM_FCD = 6,
|
|
jpayne@69
|
155
|
|
jpayne@69
|
156 /** One more than the highest normalization mode constant. @deprecated ICU 56 Use unorm2.h instead. */
|
|
jpayne@69
|
157 UNORM_MODE_COUNT
|
|
jpayne@69
|
158 } UNormalizationMode;
|
|
jpayne@69
|
159
|
|
jpayne@69
|
160 #endif // U_FORCE_HIDE_DEPRECATED_API
|
|
jpayne@69
|
161
|
|
jpayne@69
|
162 #ifndef U_HIDE_DEPRECATED_API
|
|
jpayne@69
|
163
|
|
jpayne@69
|
164 /**
|
|
jpayne@69
|
165 * Constants for options flags for normalization.
|
|
jpayne@69
|
166 * Use 0 for default options,
|
|
jpayne@69
|
167 * including normalization according to the Unicode version
|
|
jpayne@69
|
168 * that is currently supported by ICU (see u_getUnicodeVersion).
|
|
jpayne@69
|
169 * @deprecated ICU 56 Use unorm2.h instead.
|
|
jpayne@69
|
170 */
|
|
jpayne@69
|
171 enum {
|
|
jpayne@69
|
172 /**
|
|
jpayne@69
|
173 * Options bit set value to select Unicode 3.2 normalization
|
|
jpayne@69
|
174 * (except NormalizationCorrections).
|
|
jpayne@69
|
175 * At most one Unicode version can be selected at a time.
|
|
jpayne@69
|
176 * @deprecated ICU 56 Use unorm2.h instead.
|
|
jpayne@69
|
177 */
|
|
jpayne@69
|
178 UNORM_UNICODE_3_2=0x20
|
|
jpayne@69
|
179 };
|
|
jpayne@69
|
180
|
|
jpayne@69
|
181 /**
|
|
jpayne@69
|
182 * Lowest-order bit number of unorm_compare() options bits corresponding to
|
|
jpayne@69
|
183 * normalization options bits.
|
|
jpayne@69
|
184 *
|
|
jpayne@69
|
185 * The options parameter for unorm_compare() uses most bits for
|
|
jpayne@69
|
186 * itself and for various comparison and folding flags.
|
|
jpayne@69
|
187 * The most significant bits, however, are shifted down and passed on
|
|
jpayne@69
|
188 * to the normalization implementation.
|
|
jpayne@69
|
189 * (That is, from unorm_compare(..., options, ...),
|
|
jpayne@69
|
190 * options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT will be passed on to the
|
|
jpayne@69
|
191 * internal normalization functions.)
|
|
jpayne@69
|
192 *
|
|
jpayne@69
|
193 * @see unorm_compare
|
|
jpayne@69
|
194 * @deprecated ICU 56 Use unorm2.h instead.
|
|
jpayne@69
|
195 */
|
|
jpayne@69
|
196 #define UNORM_COMPARE_NORM_OPTIONS_SHIFT 20
|
|
jpayne@69
|
197
|
|
jpayne@69
|
198 /**
|
|
jpayne@69
|
199 * Normalize a string.
|
|
jpayne@69
|
200 * The string will be normalized according the specified normalization mode
|
|
jpayne@69
|
201 * and options.
|
|
jpayne@69
|
202 * The source and result buffers must not be the same, nor overlap.
|
|
jpayne@69
|
203 *
|
|
jpayne@69
|
204 * @param source The string to normalize.
|
|
jpayne@69
|
205 * @param sourceLength The length of source, or -1 if NUL-terminated.
|
|
jpayne@69
|
206 * @param mode The normalization mode; one of UNORM_NONE,
|
|
jpayne@69
|
207 * UNORM_NFD, UNORM_NFC, UNORM_NFKC, UNORM_NFKD, UNORM_DEFAULT.
|
|
jpayne@69
|
208 * @param options The normalization options, ORed together (0 for no options).
|
|
jpayne@69
|
209 * @param result A pointer to a buffer to receive the result string.
|
|
jpayne@69
|
210 * The result string is NUL-terminated if possible.
|
|
jpayne@69
|
211 * @param resultLength The maximum size of result.
|
|
jpayne@69
|
212 * @param status A pointer to a UErrorCode to receive any errors.
|
|
jpayne@69
|
213 * @return The total buffer size needed; if greater than resultLength,
|
|
jpayne@69
|
214 * the output was truncated, and the error code is set to U_BUFFER_OVERFLOW_ERROR.
|
|
jpayne@69
|
215 * @deprecated ICU 56 Use unorm2.h instead.
|
|
jpayne@69
|
216 */
|
|
jpayne@69
|
217 U_DEPRECATED int32_t U_EXPORT2
|
|
jpayne@69
|
218 unorm_normalize(const UChar *source, int32_t sourceLength,
|
|
jpayne@69
|
219 UNormalizationMode mode, int32_t options,
|
|
jpayne@69
|
220 UChar *result, int32_t resultLength,
|
|
jpayne@69
|
221 UErrorCode *status);
|
|
jpayne@69
|
222
|
|
jpayne@69
|
223 /**
|
|
jpayne@69
|
224 * Performing quick check on a string, to quickly determine if the string is
|
|
jpayne@69
|
225 * in a particular normalization format.
|
|
jpayne@69
|
226 * Three types of result can be returned UNORM_YES, UNORM_NO or
|
|
jpayne@69
|
227 * UNORM_MAYBE. Result UNORM_YES indicates that the argument
|
|
jpayne@69
|
228 * string is in the desired normalized format, UNORM_NO determines that
|
|
jpayne@69
|
229 * argument string is not in the desired normalized format. A
|
|
jpayne@69
|
230 * UNORM_MAYBE result indicates that a more thorough check is required,
|
|
jpayne@69
|
231 * the user may have to put the string in its normalized form and compare the
|
|
jpayne@69
|
232 * results.
|
|
jpayne@69
|
233 *
|
|
jpayne@69
|
234 * @param source string for determining if it is in a normalized format
|
|
jpayne@69
|
235 * @param sourcelength length of source to test, or -1 if NUL-terminated
|
|
jpayne@69
|
236 * @param mode which normalization form to test for
|
|
jpayne@69
|
237 * @param status a pointer to a UErrorCode to receive any errors
|
|
jpayne@69
|
238 * @return UNORM_YES, UNORM_NO or UNORM_MAYBE
|
|
jpayne@69
|
239 *
|
|
jpayne@69
|
240 * @see unorm_isNormalized
|
|
jpayne@69
|
241 * @deprecated ICU 56 Use unorm2.h instead.
|
|
jpayne@69
|
242 */
|
|
jpayne@69
|
243 U_DEPRECATED UNormalizationCheckResult U_EXPORT2
|
|
jpayne@69
|
244 unorm_quickCheck(const UChar *source, int32_t sourcelength,
|
|
jpayne@69
|
245 UNormalizationMode mode,
|
|
jpayne@69
|
246 UErrorCode *status);
|
|
jpayne@69
|
247
|
|
jpayne@69
|
248 /**
|
|
jpayne@69
|
249 * Performing quick check on a string; same as unorm_quickCheck but
|
|
jpayne@69
|
250 * takes an extra options parameter like most normalization functions.
|
|
jpayne@69
|
251 *
|
|
jpayne@69
|
252 * @param src String that is to be tested if it is in a normalization format.
|
|
jpayne@69
|
253 * @param srcLength Length of source to test, or -1 if NUL-terminated.
|
|
jpayne@69
|
254 * @param mode Which normalization form to test for.
|
|
jpayne@69
|
255 * @param options The normalization options, ORed together (0 for no options).
|
|
jpayne@69
|
256 * @param pErrorCode ICU error code in/out parameter.
|
|
jpayne@69
|
257 * Must fulfill U_SUCCESS before the function call.
|
|
jpayne@69
|
258 * @return UNORM_YES, UNORM_NO or UNORM_MAYBE
|
|
jpayne@69
|
259 *
|
|
jpayne@69
|
260 * @see unorm_quickCheck
|
|
jpayne@69
|
261 * @see unorm_isNormalized
|
|
jpayne@69
|
262 * @deprecated ICU 56 Use unorm2.h instead.
|
|
jpayne@69
|
263 */
|
|
jpayne@69
|
264 U_DEPRECATED UNormalizationCheckResult U_EXPORT2
|
|
jpayne@69
|
265 unorm_quickCheckWithOptions(const UChar *src, int32_t srcLength,
|
|
jpayne@69
|
266 UNormalizationMode mode, int32_t options,
|
|
jpayne@69
|
267 UErrorCode *pErrorCode);
|
|
jpayne@69
|
268
|
|
jpayne@69
|
269 /**
|
|
jpayne@69
|
270 * Test if a string is in a given normalization form.
|
|
jpayne@69
|
271 * This is semantically equivalent to source.equals(normalize(source, mode)) .
|
|
jpayne@69
|
272 *
|
|
jpayne@69
|
273 * Unlike unorm_quickCheck(), this function returns a definitive result,
|
|
jpayne@69
|
274 * never a "maybe".
|
|
jpayne@69
|
275 * For NFD, NFKD, and FCD, both functions work exactly the same.
|
|
jpayne@69
|
276 * For NFC and NFKC where quickCheck may return "maybe", this function will
|
|
jpayne@69
|
277 * perform further tests to arrive at a TRUE/FALSE result.
|
|
jpayne@69
|
278 *
|
|
jpayne@69
|
279 * @param src String that is to be tested if it is in a normalization format.
|
|
jpayne@69
|
280 * @param srcLength Length of source to test, or -1 if NUL-terminated.
|
|
jpayne@69
|
281 * @param mode Which normalization form to test for.
|
|
jpayne@69
|
282 * @param pErrorCode ICU error code in/out parameter.
|
|
jpayne@69
|
283 * Must fulfill U_SUCCESS before the function call.
|
|
jpayne@69
|
284 * @return Boolean value indicating whether the source string is in the
|
|
jpayne@69
|
285 * "mode" normalization form.
|
|
jpayne@69
|
286 *
|
|
jpayne@69
|
287 * @see unorm_quickCheck
|
|
jpayne@69
|
288 * @deprecated ICU 56 Use unorm2.h instead.
|
|
jpayne@69
|
289 */
|
|
jpayne@69
|
290 U_DEPRECATED UBool U_EXPORT2
|
|
jpayne@69
|
291 unorm_isNormalized(const UChar *src, int32_t srcLength,
|
|
jpayne@69
|
292 UNormalizationMode mode,
|
|
jpayne@69
|
293 UErrorCode *pErrorCode);
|
|
jpayne@69
|
294
|
|
jpayne@69
|
295 /**
|
|
jpayne@69
|
296 * Test if a string is in a given normalization form; same as unorm_isNormalized but
|
|
jpayne@69
|
297 * takes an extra options parameter like most normalization functions.
|
|
jpayne@69
|
298 *
|
|
jpayne@69
|
299 * @param src String that is to be tested if it is in a normalization format.
|
|
jpayne@69
|
300 * @param srcLength Length of source to test, or -1 if NUL-terminated.
|
|
jpayne@69
|
301 * @param mode Which normalization form to test for.
|
|
jpayne@69
|
302 * @param options The normalization options, ORed together (0 for no options).
|
|
jpayne@69
|
303 * @param pErrorCode ICU error code in/out parameter.
|
|
jpayne@69
|
304 * Must fulfill U_SUCCESS before the function call.
|
|
jpayne@69
|
305 * @return Boolean value indicating whether the source string is in the
|
|
jpayne@69
|
306 * "mode/options" normalization form.
|
|
jpayne@69
|
307 *
|
|
jpayne@69
|
308 * @see unorm_quickCheck
|
|
jpayne@69
|
309 * @see unorm_isNormalized
|
|
jpayne@69
|
310 * @deprecated ICU 56 Use unorm2.h instead.
|
|
jpayne@69
|
311 */
|
|
jpayne@69
|
312 U_DEPRECATED UBool U_EXPORT2
|
|
jpayne@69
|
313 unorm_isNormalizedWithOptions(const UChar *src, int32_t srcLength,
|
|
jpayne@69
|
314 UNormalizationMode mode, int32_t options,
|
|
jpayne@69
|
315 UErrorCode *pErrorCode);
|
|
jpayne@69
|
316
|
|
jpayne@69
|
317 /**
|
|
jpayne@69
|
318 * Iterative normalization forward.
|
|
jpayne@69
|
319 * This function (together with unorm_previous) is somewhat
|
|
jpayne@69
|
320 * similar to the C++ Normalizer class (see its non-static functions).
|
|
jpayne@69
|
321 *
|
|
jpayne@69
|
322 * Iterative normalization is useful when only a small portion of a longer
|
|
jpayne@69
|
323 * string/text needs to be processed.
|
|
jpayne@69
|
324 *
|
|
jpayne@69
|
325 * For example, the likelihood may be high that processing the first 10% of some
|
|
jpayne@69
|
326 * text will be sufficient to find certain data.
|
|
jpayne@69
|
327 * Another example: When one wants to concatenate two normalized strings and get a
|
|
jpayne@69
|
328 * normalized result, it is much more efficient to normalize just a small part of
|
|
jpayne@69
|
329 * the result around the concatenation place instead of re-normalizing everything.
|
|
jpayne@69
|
330 *
|
|
jpayne@69
|
331 * The input text is an instance of the C character iteration API UCharIterator.
|
|
jpayne@69
|
332 * It may wrap around a simple string, a CharacterIterator, a Replaceable, or any
|
|
jpayne@69
|
333 * other kind of text object.
|
|
jpayne@69
|
334 *
|
|
jpayne@69
|
335 * If a buffer overflow occurs, then the caller needs to reset the iterator to the
|
|
jpayne@69
|
336 * old index and call the function again with a larger buffer - if the caller cares
|
|
jpayne@69
|
337 * for the actual output.
|
|
jpayne@69
|
338 * Regardless of the output buffer, the iterator will always be moved to the next
|
|
jpayne@69
|
339 * normalization boundary.
|
|
jpayne@69
|
340 *
|
|
jpayne@69
|
341 * This function (like unorm_previous) serves two purposes:
|
|
jpayne@69
|
342 *
|
|
jpayne@69
|
343 * 1) To find the next boundary so that the normalization of the part of the text
|
|
jpayne@69
|
344 * from the current position to that boundary does not affect and is not affected
|
|
jpayne@69
|
345 * by the part of the text beyond that boundary.
|
|
jpayne@69
|
346 *
|
|
jpayne@69
|
347 * 2) To normalize the text up to the boundary.
|
|
jpayne@69
|
348 *
|
|
jpayne@69
|
349 * The second step is optional, per the doNormalize parameter.
|
|
jpayne@69
|
350 * It is omitted for operations like string concatenation, where the two adjacent
|
|
jpayne@69
|
351 * string ends need to be normalized together.
|
|
jpayne@69
|
352 * In such a case, the output buffer will just contain a copy of the text up to the
|
|
jpayne@69
|
353 * boundary.
|
|
jpayne@69
|
354 *
|
|
jpayne@69
|
355 * pNeededToNormalize is an output-only parameter. Its output value is only defined
|
|
jpayne@69
|
356 * if normalization was requested (doNormalize) and successful (especially, no
|
|
jpayne@69
|
357 * buffer overflow).
|
|
jpayne@69
|
358 * It is useful for operations like a normalizing transliterator, where one would
|
|
jpayne@69
|
359 * not want to replace a piece of text if it is not modified.
|
|
jpayne@69
|
360 *
|
|
jpayne@69
|
361 * If doNormalize==TRUE and pNeededToNormalize!=NULL then *pNeeded... is set TRUE
|
|
jpayne@69
|
362 * if the normalization was necessary.
|
|
jpayne@69
|
363 *
|
|
jpayne@69
|
364 * If doNormalize==FALSE then *pNeededToNormalize will be set to FALSE.
|
|
jpayne@69
|
365 *
|
|
jpayne@69
|
366 * If the buffer overflows, then *pNeededToNormalize will be undefined;
|
|
jpayne@69
|
367 * essentially, whenever U_FAILURE is true (like in buffer overflows), this result
|
|
jpayne@69
|
368 * will be undefined.
|
|
jpayne@69
|
369 *
|
|
jpayne@69
|
370 * @param src The input text in the form of a C character iterator.
|
|
jpayne@69
|
371 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.
|
|
jpayne@69
|
372 * @param destCapacity The number of UChars that fit into dest.
|
|
jpayne@69
|
373 * @param mode The normalization mode.
|
|
jpayne@69
|
374 * @param options The normalization options, ORed together (0 for no options).
|
|
jpayne@69
|
375 * @param doNormalize Indicates if the source text up to the next boundary
|
|
jpayne@69
|
376 * is to be normalized (TRUE) or just copied (FALSE).
|
|
jpayne@69
|
377 * @param pNeededToNormalize Output flag indicating if the normalization resulted in
|
|
jpayne@69
|
378 * different text from the input.
|
|
jpayne@69
|
379 * Not defined if an error occurs including buffer overflow.
|
|
jpayne@69
|
380 * Always FALSE if !doNormalize.
|
|
jpayne@69
|
381 * @param pErrorCode ICU error code in/out parameter.
|
|
jpayne@69
|
382 * Must fulfill U_SUCCESS before the function call.
|
|
jpayne@69
|
383 * @return Length of output (number of UChars) when successful or buffer overflow.
|
|
jpayne@69
|
384 *
|
|
jpayne@69
|
385 * @see unorm_previous
|
|
jpayne@69
|
386 * @see unorm_normalize
|
|
jpayne@69
|
387 *
|
|
jpayne@69
|
388 * @deprecated ICU 56 Use unorm2.h instead.
|
|
jpayne@69
|
389 */
|
|
jpayne@69
|
390 U_DEPRECATED int32_t U_EXPORT2
|
|
jpayne@69
|
391 unorm_next(UCharIterator *src,
|
|
jpayne@69
|
392 UChar *dest, int32_t destCapacity,
|
|
jpayne@69
|
393 UNormalizationMode mode, int32_t options,
|
|
jpayne@69
|
394 UBool doNormalize, UBool *pNeededToNormalize,
|
|
jpayne@69
|
395 UErrorCode *pErrorCode);
|
|
jpayne@69
|
396
|
|
jpayne@69
|
397 /**
|
|
jpayne@69
|
398 * Iterative normalization backward.
|
|
jpayne@69
|
399 * This function (together with unorm_next) is somewhat
|
|
jpayne@69
|
400 * similar to the C++ Normalizer class (see its non-static functions).
|
|
jpayne@69
|
401 * For all details see unorm_next.
|
|
jpayne@69
|
402 *
|
|
jpayne@69
|
403 * @param src The input text in the form of a C character iterator.
|
|
jpayne@69
|
404 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.
|
|
jpayne@69
|
405 * @param destCapacity The number of UChars that fit into dest.
|
|
jpayne@69
|
406 * @param mode The normalization mode.
|
|
jpayne@69
|
407 * @param options The normalization options, ORed together (0 for no options).
|
|
jpayne@69
|
408 * @param doNormalize Indicates if the source text up to the next boundary
|
|
jpayne@69
|
409 * is to be normalized (TRUE) or just copied (FALSE).
|
|
jpayne@69
|
410 * @param pNeededToNormalize Output flag indicating if the normalization resulted in
|
|
jpayne@69
|
411 * different text from the input.
|
|
jpayne@69
|
412 * Not defined if an error occurs including buffer overflow.
|
|
jpayne@69
|
413 * Always FALSE if !doNormalize.
|
|
jpayne@69
|
414 * @param pErrorCode ICU error code in/out parameter.
|
|
jpayne@69
|
415 * Must fulfill U_SUCCESS before the function call.
|
|
jpayne@69
|
416 * @return Length of output (number of UChars) when successful or buffer overflow.
|
|
jpayne@69
|
417 *
|
|
jpayne@69
|
418 * @see unorm_next
|
|
jpayne@69
|
419 * @see unorm_normalize
|
|
jpayne@69
|
420 *
|
|
jpayne@69
|
421 * @deprecated ICU 56 Use unorm2.h instead.
|
|
jpayne@69
|
422 */
|
|
jpayne@69
|
423 U_DEPRECATED int32_t U_EXPORT2
|
|
jpayne@69
|
424 unorm_previous(UCharIterator *src,
|
|
jpayne@69
|
425 UChar *dest, int32_t destCapacity,
|
|
jpayne@69
|
426 UNormalizationMode mode, int32_t options,
|
|
jpayne@69
|
427 UBool doNormalize, UBool *pNeededToNormalize,
|
|
jpayne@69
|
428 UErrorCode *pErrorCode);
|
|
jpayne@69
|
429
|
|
jpayne@69
|
430 /**
|
|
jpayne@69
|
431 * Concatenate normalized strings, making sure that the result is normalized as well.
|
|
jpayne@69
|
432 *
|
|
jpayne@69
|
433 * If both the left and the right strings are in
|
|
jpayne@69
|
434 * the normalization form according to "mode/options",
|
|
jpayne@69
|
435 * then the result will be
|
|
jpayne@69
|
436 *
|
|
jpayne@69
|
437 * \code
|
|
jpayne@69
|
438 * dest=normalize(left+right, mode, options)
|
|
jpayne@69
|
439 * \endcode
|
|
jpayne@69
|
440 *
|
|
jpayne@69
|
441 * With the input strings already being normalized,
|
|
jpayne@69
|
442 * this function will use unorm_next() and unorm_previous()
|
|
jpayne@69
|
443 * to find the adjacent end pieces of the input strings.
|
|
jpayne@69
|
444 * Only the concatenation of these end pieces will be normalized and
|
|
jpayne@69
|
445 * then concatenated with the remaining parts of the input strings.
|
|
jpayne@69
|
446 *
|
|
jpayne@69
|
447 * It is allowed to have dest==left to avoid copying the entire left string.
|
|
jpayne@69
|
448 *
|
|
jpayne@69
|
449 * @param left Left source string, may be same as dest.
|
|
jpayne@69
|
450 * @param leftLength Length of left source string, or -1 if NUL-terminated.
|
|
jpayne@69
|
451 * @param right Right source string. Must not be the same as dest, nor overlap.
|
|
jpayne@69
|
452 * @param rightLength Length of right source string, or -1 if NUL-terminated.
|
|
jpayne@69
|
453 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.
|
|
jpayne@69
|
454 * @param destCapacity The number of UChars that fit into dest.
|
|
jpayne@69
|
455 * @param mode The normalization mode.
|
|
jpayne@69
|
456 * @param options The normalization options, ORed together (0 for no options).
|
|
jpayne@69
|
457 * @param pErrorCode ICU error code in/out parameter.
|
|
jpayne@69
|
458 * Must fulfill U_SUCCESS before the function call.
|
|
jpayne@69
|
459 * @return Length of output (number of UChars) when successful or buffer overflow.
|
|
jpayne@69
|
460 *
|
|
jpayne@69
|
461 * @see unorm_normalize
|
|
jpayne@69
|
462 * @see unorm_next
|
|
jpayne@69
|
463 * @see unorm_previous
|
|
jpayne@69
|
464 *
|
|
jpayne@69
|
465 * @deprecated ICU 56 Use unorm2.h instead.
|
|
jpayne@69
|
466 */
|
|
jpayne@69
|
467 U_DEPRECATED int32_t U_EXPORT2
|
|
jpayne@69
|
468 unorm_concatenate(const UChar *left, int32_t leftLength,
|
|
jpayne@69
|
469 const UChar *right, int32_t rightLength,
|
|
jpayne@69
|
470 UChar *dest, int32_t destCapacity,
|
|
jpayne@69
|
471 UNormalizationMode mode, int32_t options,
|
|
jpayne@69
|
472 UErrorCode *pErrorCode);
|
|
jpayne@69
|
473
|
|
jpayne@69
|
474 #endif /* U_HIDE_DEPRECATED_API */
|
|
jpayne@69
|
475 #endif /* #if !UCONFIG_NO_NORMALIZATION */
|
|
jpayne@69
|
476 #endif
|
