Mercurial > repos > rliterman > csp2

annotate CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/include/unicode/ucsdet.h @ 69:33d812a61356

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 17:55:14 -0400
parents
children
rev   line source
jpayne@69 1 // © 2016 and later: Unicode, Inc. and others.
jpayne@69 2 // License & terms of use: http://www.unicode.org/copyright.html
jpayne@69 3 /*
jpayne@69 4 **********************************************************************
jpayne@69 5 * Copyright (C) 2005-2013, International Business Machines
jpayne@69 6 * Corporation and others. All Rights Reserved.
jpayne@69 7 **********************************************************************
jpayne@69 8 * file name: ucsdet.h
jpayne@69 9 * encoding: UTF-8
jpayne@69 10 * indentation:4
jpayne@69 11 *
jpayne@69 12 * created on: 2005Aug04
jpayne@69 13 * created by: Andy Heninger
jpayne@69 14 *
jpayne@69 15 * ICU Character Set Detection, API for C
jpayne@69 16 *
jpayne@69 17 * Draft version 18 Oct 2005
jpayne@69 18 *
jpayne@69 19 */
jpayne@69 20
jpayne@69 21 #ifndef __UCSDET_H
jpayne@69 22 #define __UCSDET_H
jpayne@69 23
jpayne@69 24 #include "unicode/utypes.h"
jpayne@69 25
jpayne@69 26 #if !UCONFIG_NO_CONVERSION
jpayne@69 27
jpayne@69 28 #include "unicode/localpointer.h"
jpayne@69 29 #include "unicode/uenum.h"
jpayne@69 30
jpayne@69 31 /**
jpayne@69 32 * \file
jpayne@69 33 * \brief C API: Charset Detection API
jpayne@69 34 *
jpayne@69 35 * This API provides a facility for detecting the
jpayne@69 36 * charset or encoding of character data in an unknown text format.
jpayne@69 37 * The input data can be from an array of bytes.
jpayne@69 38 * <p>
jpayne@69 39 * Character set detection is at best an imprecise operation. The detection
jpayne@69 40 * process will attempt to identify the charset that best matches the characteristics
jpayne@69 41 * of the byte data, but the process is partly statistical in nature, and
jpayne@69 42 * the results can not be guaranteed to always be correct.
jpayne@69 43 * <p>
jpayne@69 44 * For best accuracy in charset detection, the input data should be primarily
jpayne@69 45 * in a single language, and a minimum of a few hundred bytes worth of plain text
jpayne@69 46 * in the language are needed. The detection process will attempt to
jpayne@69 47 * ignore html or xml style markup that could otherwise obscure the content.
jpayne@69 48 * <p>
jpayne@69 49 * An alternative to the ICU Charset Detector is the
jpayne@69 50 * Compact Encoding Detector, https://github.com/google/compact_enc_det.
jpayne@69 51 * It often gives more accurate results, especially with short input samples.
jpayne@69 52 */
jpayne@69 53
jpayne@69 54
jpayne@69 55 struct UCharsetDetector;
jpayne@69 56 /**
jpayne@69 57 * Structure representing a charset detector
jpayne@69 58 * @stable ICU 3.6
jpayne@69 59 */
jpayne@69 60 typedef struct UCharsetDetector UCharsetDetector;
jpayne@69 61
jpayne@69 62 struct UCharsetMatch;
jpayne@69 63 /**
jpayne@69 64 * Opaque structure representing a match that was identified
jpayne@69 65 * from a charset detection operation.
jpayne@69 66 * @stable ICU 3.6
jpayne@69 67 */
jpayne@69 68 typedef struct UCharsetMatch UCharsetMatch;
jpayne@69 69
jpayne@69 70 /**
jpayne@69 71 * Open a charset detector.
jpayne@69 72 *
jpayne@69 73 * @param status Any error conditions occurring during the open
jpayne@69 74 * operation are reported back in this variable.
jpayne@69 75 * @return the newly opened charset detector.
jpayne@69 76 * @stable ICU 3.6
jpayne@69 77 */
jpayne@69 78 U_STABLE UCharsetDetector * U_EXPORT2
jpayne@69 79 ucsdet_open(UErrorCode *status);
jpayne@69 80
jpayne@69 81 /**
jpayne@69 82 * Close a charset detector. All storage and any other resources
jpayne@69 83 * owned by this charset detector will be released. Failure to
jpayne@69 84 * close a charset detector when finished with it can result in
jpayne@69 85 * memory leaks in the application.
jpayne@69 86 *
jpayne@69 87 * @param ucsd The charset detector to be closed.
jpayne@69 88 * @stable ICU 3.6
jpayne@69 89 */
jpayne@69 90 U_STABLE void U_EXPORT2
jpayne@69 91 ucsdet_close(UCharsetDetector *ucsd);
jpayne@69 92
jpayne@69 93 #if U_SHOW_CPLUSPLUS_API
jpayne@69 94
jpayne@69 95 U_NAMESPACE_BEGIN
jpayne@69 96
jpayne@69 97 /**
jpayne@69 98 * \class LocalUCharsetDetectorPointer
jpayne@69 99 * "Smart pointer" class, closes a UCharsetDetector via ucsdet_close().
jpayne@69 100 * For most methods see the LocalPointerBase base class.
jpayne@69 101 *
jpayne@69 102 * @see LocalPointerBase
jpayne@69 103 * @see LocalPointer
jpayne@69 104 * @stable ICU 4.4
jpayne@69 105 */
jpayne@69 106 U_DEFINE_LOCAL_OPEN_POINTER(LocalUCharsetDetectorPointer, UCharsetDetector, ucsdet_close);
jpayne@69 107
jpayne@69 108 U_NAMESPACE_END
jpayne@69 109
jpayne@69 110 #endif
jpayne@69 111
jpayne@69 112 /**
jpayne@69 113 * Set the input byte data whose charset is to detected.
jpayne@69 114 *
jpayne@69 115 * Ownership of the input text byte array remains with the caller.
jpayne@69 116 * The input string must not be altered or deleted until the charset
jpayne@69 117 * detector is either closed or reset to refer to different input text.
jpayne@69 118 *
jpayne@69 119 * @param ucsd the charset detector to be used.
jpayne@69 120 * @param textIn the input text of unknown encoding. .
jpayne@69 121 * @param len the length of the input text, or -1 if the text
jpayne@69 122 * is NUL terminated.
jpayne@69 123 * @param status any error conditions are reported back in this variable.
jpayne@69 124 *
jpayne@69 125 * @stable ICU 3.6
jpayne@69 126 */
jpayne@69 127 U_STABLE void U_EXPORT2
jpayne@69 128 ucsdet_setText(UCharsetDetector *ucsd, const char *textIn, int32_t len, UErrorCode *status);
jpayne@69 129
jpayne@69 130
jpayne@69 131 /** Set the declared encoding for charset detection.
jpayne@69 132 * The declared encoding of an input text is an encoding obtained
jpayne@69 133 * by the user from an http header or xml declaration or similar source that
jpayne@69 134 * can be provided as an additional hint to the charset detector.
jpayne@69 135 *
jpayne@69 136 * How and whether the declared encoding will be used during the
jpayne@69 137 * detection process is TBD.
jpayne@69 138 *
jpayne@69 139 * @param ucsd the charset detector to be used.
jpayne@69 140 * @param encoding an encoding for the current data obtained from
jpayne@69 141 * a header or declaration or other source outside
jpayne@69 142 * of the byte data itself.
jpayne@69 143 * @param length the length of the encoding name, or -1 if the name string
jpayne@69 144 * is NUL terminated.
jpayne@69 145 * @param status any error conditions are reported back in this variable.
jpayne@69 146 *
jpayne@69 147 * @stable ICU 3.6
jpayne@69 148 */
jpayne@69 149 U_STABLE void U_EXPORT2
jpayne@69 150 ucsdet_setDeclaredEncoding(UCharsetDetector *ucsd, const char *encoding, int32_t length, UErrorCode *status);
jpayne@69 151
jpayne@69 152
jpayne@69 153 /**
jpayne@69 154 * Return the charset that best matches the supplied input data.
jpayne@69 155 *
jpayne@69 156 * Note though, that because the detection
jpayne@69 157 * only looks at the start of the input data,
jpayne@69 158 * there is a possibility that the returned charset will fail to handle
jpayne@69 159 * the full set of input data.
jpayne@69 160 * <p>
jpayne@69 161 * The returned UCharsetMatch object is owned by the UCharsetDetector.
jpayne@69 162 * It will remain valid until the detector input is reset, or until
jpayne@69 163 * the detector is closed.
jpayne@69 164 * <p>
jpayne@69 165 * The function will fail if
jpayne@69 166 * <ul>
jpayne@69 167 * <li>no charset appears to match the data.</li>
jpayne@69 168 * <li>no input text has been provided</li>
jpayne@69 169 * </ul>
jpayne@69 170 *
jpayne@69 171 * @param ucsd the charset detector to be used.
jpayne@69 172 * @param status any error conditions are reported back in this variable.
jpayne@69 173 * @return a UCharsetMatch representing the best matching charset,
jpayne@69 174 * or NULL if no charset matches the byte data.
jpayne@69 175 *
jpayne@69 176 * @stable ICU 3.6
jpayne@69 177 */
jpayne@69 178 U_STABLE const UCharsetMatch * U_EXPORT2
jpayne@69 179 ucsdet_detect(UCharsetDetector *ucsd, UErrorCode *status);
jpayne@69 180
jpayne@69 181
jpayne@69 182 /**
jpayne@69 183 * Find all charset matches that appear to be consistent with the input,
jpayne@69 184 * returning an array of results. The results are ordered with the
jpayne@69 185 * best quality match first.
jpayne@69 186 *
jpayne@69 187 * Because the detection only looks at a limited amount of the
jpayne@69 188 * input byte data, some of the returned charsets may fail to handle
jpayne@69 189 * the all of input data.
jpayne@69 190 * <p>
jpayne@69 191 * The returned UCharsetMatch objects are owned by the UCharsetDetector.
jpayne@69 192 * They will remain valid until the detector is closed or modified
jpayne@69 193 *
jpayne@69 194 * <p>
jpayne@69 195 * Return an error if
jpayne@69 196 * <ul>
jpayne@69 197 * <li>no charsets appear to match the input data.</li>
jpayne@69 198 * <li>no input text has been provided</li>
jpayne@69 199 * </ul>
jpayne@69 200 *
jpayne@69 201 * @param ucsd the charset detector to be used.
jpayne@69 202 * @param matchesFound pointer to a variable that will be set to the
jpayne@69 203 * number of charsets identified that are consistent with
jpayne@69 204 * the input data. Output only.
jpayne@69 205 * @param status any error conditions are reported back in this variable.
jpayne@69 206 * @return A pointer to an array of pointers to UCharSetMatch objects.
jpayne@69 207 * This array, and the UCharSetMatch instances to which it refers,
jpayne@69 208 * are owned by the UCharsetDetector, and will remain valid until
jpayne@69 209 * the detector is closed or modified.
jpayne@69 210 * @stable ICU 3.6
jpayne@69 211 */
jpayne@69 212 U_STABLE const UCharsetMatch ** U_EXPORT2
jpayne@69 213 ucsdet_detectAll(UCharsetDetector *ucsd, int32_t *matchesFound, UErrorCode *status);
jpayne@69 214
jpayne@69 215
jpayne@69 216
jpayne@69 217 /**
jpayne@69 218 * Get the name of the charset represented by a UCharsetMatch.
jpayne@69 219 *
jpayne@69 220 * The storage for the returned name string is owned by the
jpayne@69 221 * UCharsetMatch, and will remain valid while the UCharsetMatch
jpayne@69 222 * is valid.
jpayne@69 223 *
jpayne@69 224 * The name returned is suitable for use with the ICU conversion APIs.
jpayne@69 225 *
jpayne@69 226 * @param ucsm The charset match object.
jpayne@69 227 * @param status Any error conditions are reported back in this variable.
jpayne@69 228 * @return The name of the matching charset.
jpayne@69 229 *
jpayne@69 230 * @stable ICU 3.6
jpayne@69 231 */
jpayne@69 232 U_STABLE const char * U_EXPORT2
jpayne@69 233 ucsdet_getName(const UCharsetMatch *ucsm, UErrorCode *status);
jpayne@69 234
jpayne@69 235 /**
jpayne@69 236 * Get a confidence number for the quality of the match of the byte
jpayne@69 237 * data with the charset. Confidence numbers range from zero to 100,
jpayne@69 238 * with 100 representing complete confidence and zero representing
jpayne@69 239 * no confidence.
jpayne@69 240 *
jpayne@69 241 * The confidence values are somewhat arbitrary. They define an
jpayne@69 242 * an ordering within the results for any single detection operation
jpayne@69 243 * but are not generally comparable between the results for different input.
jpayne@69 244 *
jpayne@69 245 * A confidence value of ten does have a general meaning - it is used
jpayne@69 246 * for charsets that can represent the input data, but for which there
jpayne@69 247 * is no other indication that suggests that the charset is the correct one.
jpayne@69 248 * Pure 7 bit ASCII data, for example, is compatible with a
jpayne@69 249 * great many charsets, most of which will appear as possible matches
jpayne@69 250 * with a confidence of 10.
jpayne@69 251 *
jpayne@69 252 * @param ucsm The charset match object.
jpayne@69 253 * @param status Any error conditions are reported back in this variable.
jpayne@69 254 * @return A confidence number for the charset match.
jpayne@69 255 *
jpayne@69 256 * @stable ICU 3.6
jpayne@69 257 */
jpayne@69 258 U_STABLE int32_t U_EXPORT2
jpayne@69 259 ucsdet_getConfidence(const UCharsetMatch *ucsm, UErrorCode *status);
jpayne@69 260
jpayne@69 261 /**
jpayne@69 262 * Get the RFC 3066 code for the language of the input data.
jpayne@69 263 *
jpayne@69 264 * The Charset Detection service is intended primarily for detecting
jpayne@69 265 * charsets, not language. For some, but not all, charsets, a language is
jpayne@69 266 * identified as a byproduct of the detection process, and that is what
jpayne@69 267 * is returned by this function.
jpayne@69 268 *
jpayne@69 269 * CAUTION:
jpayne@69 270 * 1. Language information is not available for input data encoded in
jpayne@69 271 * all charsets. In particular, no language is identified
jpayne@69 272 * for UTF-8 input data.
jpayne@69 273 *
jpayne@69 274 * 2. Closely related languages may sometimes be confused.
jpayne@69 275 *
jpayne@69 276 * If more accurate language detection is required, a linguistic
jpayne@69 277 * analysis package should be used.
jpayne@69 278 *
jpayne@69 279 * The storage for the returned name string is owned by the
jpayne@69 280 * UCharsetMatch, and will remain valid while the UCharsetMatch
jpayne@69 281 * is valid.
jpayne@69 282 *
jpayne@69 283 * @param ucsm The charset match object.
jpayne@69 284 * @param status Any error conditions are reported back in this variable.
jpayne@69 285 * @return The RFC 3066 code for the language of the input data, or
jpayne@69 286 * an empty string if the language could not be determined.
jpayne@69 287 *
jpayne@69 288 * @stable ICU 3.6
jpayne@69 289 */
jpayne@69 290 U_STABLE const char * U_EXPORT2
jpayne@69 291 ucsdet_getLanguage(const UCharsetMatch *ucsm, UErrorCode *status);
jpayne@69 292
jpayne@69 293
jpayne@69 294 /**
jpayne@69 295 * Get the entire input text as a UChar string, placing it into
jpayne@69 296 * a caller-supplied buffer. A terminating
jpayne@69 297 * NUL character will be appended to the buffer if space is available.
jpayne@69 298 *
jpayne@69 299 * The number of UChars in the output string, not including the terminating
jpayne@69 300 * NUL, is returned.
jpayne@69 301 *
jpayne@69 302 * If the supplied buffer is smaller than required to hold the output,
jpayne@69 303 * the contents of the buffer are undefined. The full output string length
jpayne@69 304 * (in UChars) is returned as always, and can be used to allocate a buffer
jpayne@69 305 * of the correct size.
jpayne@69 306 *
jpayne@69 307 *
jpayne@69 308 * @param ucsm The charset match object.
jpayne@69 309 * @param buf A UChar buffer to be filled with the converted text data.
jpayne@69 310 * @param cap The capacity of the buffer in UChars.
jpayne@69 311 * @param status Any error conditions are reported back in this variable.
jpayne@69 312 * @return The number of UChars in the output string.
jpayne@69 313 *
jpayne@69 314 * @stable ICU 3.6
jpayne@69 315 */
jpayne@69 316 U_STABLE int32_t U_EXPORT2
jpayne@69 317 ucsdet_getUChars(const UCharsetMatch *ucsm,
jpayne@69 318 UChar *buf, int32_t cap, UErrorCode *status);
jpayne@69 319
jpayne@69 320
jpayne@69 321
jpayne@69 322 /**
jpayne@69 323 * Get an iterator over the set of all detectable charsets -
jpayne@69 324 * over the charsets that are known to the charset detection
jpayne@69 325 * service.
jpayne@69 326 *
jpayne@69 327 * The returned UEnumeration provides access to the names of
jpayne@69 328 * the charsets.
jpayne@69 329 *
jpayne@69 330 * <p>
jpayne@69 331 * The state of the Charset detector that is passed in does not
jpayne@69 332 * affect the result of this function, but requiring a valid, open
jpayne@69 333 * charset detector as a parameter insures that the charset detection
jpayne@69 334 * service has been safely initialized and that the required detection
jpayne@69 335 * data is available.
jpayne@69 336 *
jpayne@69 337 * <p>
jpayne@69 338 * <b>Note:</b> Multiple different charset encodings in a same family may use
jpayne@69 339 * a single shared name in this implementation. For example, this method returns
jpayne@69 340 * an array including "ISO-8859-1" (ISO Latin 1), but not including "windows-1252"
jpayne@69 341 * (Windows Latin 1). However, actual detection result could be "windows-1252"
jpayne@69 342 * when the input data matches Latin 1 code points with any points only available
jpayne@69 343 * in "windows-1252".
jpayne@69 344 *
jpayne@69 345 * @param ucsd a Charset detector.
jpayne@69 346 * @param status Any error conditions are reported back in this variable.
jpayne@69 347 * @return an iterator providing access to the detectable charset names.
jpayne@69 348 * @stable ICU 3.6
jpayne@69 349 */
jpayne@69 350 U_STABLE UEnumeration * U_EXPORT2
jpayne@69 351 ucsdet_getAllDetectableCharsets(const UCharsetDetector *ucsd, UErrorCode *status);
jpayne@69 352
jpayne@69 353 /**
jpayne@69 354 * Test whether input filtering is enabled for this charset detector.
jpayne@69 355 * Input filtering removes text that appears to be HTML or xml
jpayne@69 356 * markup from the input before applying the code page detection
jpayne@69 357 * heuristics.
jpayne@69 358 *
jpayne@69 359 * @param ucsd The charset detector to check.
jpayne@69 360 * @return TRUE if filtering is enabled.
jpayne@69 361 * @stable ICU 3.6
jpayne@69 362 */
jpayne@69 363
jpayne@69 364 U_STABLE UBool U_EXPORT2
jpayne@69 365 ucsdet_isInputFilterEnabled(const UCharsetDetector *ucsd);
jpayne@69 366
jpayne@69 367
jpayne@69 368 /**
jpayne@69 369 * Enable filtering of input text. If filtering is enabled,
jpayne@69 370 * text within angle brackets ("<" and ">") will be removed
jpayne@69 371 * before detection, which will remove most HTML or xml markup.
jpayne@69 372 *
jpayne@69 373 * @param ucsd the charset detector to be modified.
jpayne@69 374 * @param filter <code>true</code> to enable input text filtering.
jpayne@69 375 * @return The previous setting.
jpayne@69 376 *
jpayne@69 377 * @stable ICU 3.6
jpayne@69 378 */
jpayne@69 379 U_STABLE UBool U_EXPORT2
jpayne@69 380 ucsdet_enableInputFilter(UCharsetDetector *ucsd, UBool filter);
jpayne@69 381
jpayne@69 382 #ifndef U_HIDE_INTERNAL_API
jpayne@69 383 /**
jpayne@69 384 * Get an iterator over the set of detectable charsets -
jpayne@69 385 * over the charsets that are enabled by the specified charset detector.
jpayne@69 386 *
jpayne@69 387 * The returned UEnumeration provides access to the names of
jpayne@69 388 * the charsets.
jpayne@69 389 *
jpayne@69 390 * @param ucsd a Charset detector.
jpayne@69 391 * @param status Any error conditions are reported back in this variable.
jpayne@69 392 * @return an iterator providing access to the detectable charset names by
jpayne@69 393 * the specified charset detector.
jpayne@69 394 * @internal
jpayne@69 395 */
jpayne@69 396 U_INTERNAL UEnumeration * U_EXPORT2
jpayne@69 397 ucsdet_getDetectableCharsets(const UCharsetDetector *ucsd, UErrorCode *status);
jpayne@69 398
jpayne@69 399 /**
jpayne@69 400 * Enable or disable individual charset encoding.
jpayne@69 401 * A name of charset encoding must be included in the names returned by
jpayne@69 402 * {@link #ucsdet_getAllDetectableCharsets()}.
jpayne@69 403 *
jpayne@69 404 * @param ucsd a Charset detector.
jpayne@69 405 * @param encoding encoding the name of charset encoding.
jpayne@69 406 * @param enabled <code>TRUE</code> to enable, or <code>FALSE</code> to disable the
jpayne@69 407 * charset encoding.
jpayne@69 408 * @param status receives the return status. When the name of charset encoding
jpayne@69 409 * is not supported, U_ILLEGAL_ARGUMENT_ERROR is set.
jpayne@69 410 * @internal
jpayne@69 411 */
jpayne@69 412 U_INTERNAL void U_EXPORT2
jpayne@69 413 ucsdet_setDetectableCharset(UCharsetDetector *ucsd, const char *encoding, UBool enabled, UErrorCode *status);
jpayne@69 414 #endif /* U_HIDE_INTERNAL_API */
jpayne@69 415
jpayne@69 416 #endif
jpayne@69 417 #endif /* __UCSDET_H */
jpayne@69 418
jpayne@69 419