jpayne@69: // © 2016 and later: Unicode, Inc. and others. jpayne@69: // License & terms of use: http://www.unicode.org/copyright.html jpayne@69: /* jpayne@69: ********************************************************************** jpayne@69: * Copyright (C) 2005-2013, International Business Machines jpayne@69: * Corporation and others. All Rights Reserved. jpayne@69: ********************************************************************** jpayne@69: * file name: ucsdet.h jpayne@69: * encoding: UTF-8 jpayne@69: * indentation:4 jpayne@69: * jpayne@69: * created on: 2005Aug04 jpayne@69: * created by: Andy Heninger jpayne@69: * jpayne@69: * ICU Character Set Detection, API for C jpayne@69: * jpayne@69: * Draft version 18 Oct 2005 jpayne@69: * jpayne@69: */ jpayne@69: jpayne@69: #ifndef __UCSDET_H jpayne@69: #define __UCSDET_H jpayne@69: jpayne@69: #include "unicode/utypes.h" jpayne@69: jpayne@69: #if !UCONFIG_NO_CONVERSION jpayne@69: jpayne@69: #include "unicode/localpointer.h" jpayne@69: #include "unicode/uenum.h" jpayne@69: jpayne@69: /** jpayne@69: * \file jpayne@69: * \brief C API: Charset Detection API jpayne@69: * jpayne@69: * This API provides a facility for detecting the jpayne@69: * charset or encoding of character data in an unknown text format. jpayne@69: * The input data can be from an array of bytes. jpayne@69: *

jpayne@69: * Character set detection is at best an imprecise operation. The detection jpayne@69: * process will attempt to identify the charset that best matches the characteristics jpayne@69: * of the byte data, but the process is partly statistical in nature, and jpayne@69: * the results can not be guaranteed to always be correct. jpayne@69: *

jpayne@69: * For best accuracy in charset detection, the input data should be primarily jpayne@69: * in a single language, and a minimum of a few hundred bytes worth of plain text jpayne@69: * in the language are needed. The detection process will attempt to jpayne@69: * ignore html or xml style markup that could otherwise obscure the content. jpayne@69: *

jpayne@69: * An alternative to the ICU Charset Detector is the jpayne@69: * Compact Encoding Detector, https://github.com/google/compact_enc_det. jpayne@69: * It often gives more accurate results, especially with short input samples. jpayne@69: */ jpayne@69: jpayne@69: jpayne@69: struct UCharsetDetector; jpayne@69: /** jpayne@69: * Structure representing a charset detector jpayne@69: * @stable ICU 3.6 jpayne@69: */ jpayne@69: typedef struct UCharsetDetector UCharsetDetector; jpayne@69: jpayne@69: struct UCharsetMatch; jpayne@69: /** jpayne@69: * Opaque structure representing a match that was identified jpayne@69: * from a charset detection operation. jpayne@69: * @stable ICU 3.6 jpayne@69: */ jpayne@69: typedef struct UCharsetMatch UCharsetMatch; jpayne@69: jpayne@69: /** jpayne@69: * Open a charset detector. jpayne@69: * jpayne@69: * @param status Any error conditions occurring during the open jpayne@69: * operation are reported back in this variable. jpayne@69: * @return the newly opened charset detector. jpayne@69: * @stable ICU 3.6 jpayne@69: */ jpayne@69: U_STABLE UCharsetDetector * U_EXPORT2 jpayne@69: ucsdet_open(UErrorCode *status); jpayne@69: jpayne@69: /** jpayne@69: * Close a charset detector. All storage and any other resources jpayne@69: * owned by this charset detector will be released. Failure to jpayne@69: * close a charset detector when finished with it can result in jpayne@69: * memory leaks in the application. jpayne@69: * jpayne@69: * @param ucsd The charset detector to be closed. jpayne@69: * @stable ICU 3.6 jpayne@69: */ jpayne@69: U_STABLE void U_EXPORT2 jpayne@69: ucsdet_close(UCharsetDetector *ucsd); jpayne@69: jpayne@69: #if U_SHOW_CPLUSPLUS_API jpayne@69: jpayne@69: U_NAMESPACE_BEGIN jpayne@69: jpayne@69: /** jpayne@69: * \class LocalUCharsetDetectorPointer jpayne@69: * "Smart pointer" class, closes a UCharsetDetector via ucsdet_close(). jpayne@69: * For most methods see the LocalPointerBase base class. jpayne@69: * jpayne@69: * @see LocalPointerBase jpayne@69: * @see LocalPointer jpayne@69: * @stable ICU 4.4 jpayne@69: */ jpayne@69: U_DEFINE_LOCAL_OPEN_POINTER(LocalUCharsetDetectorPointer, UCharsetDetector, ucsdet_close); jpayne@69: jpayne@69: U_NAMESPACE_END jpayne@69: jpayne@69: #endif jpayne@69: jpayne@69: /** jpayne@69: * Set the input byte data whose charset is to detected. jpayne@69: * jpayne@69: * Ownership of the input text byte array remains with the caller. jpayne@69: * The input string must not be altered or deleted until the charset jpayne@69: * detector is either closed or reset to refer to different input text. jpayne@69: * jpayne@69: * @param ucsd the charset detector to be used. jpayne@69: * @param textIn the input text of unknown encoding. . jpayne@69: * @param len the length of the input text, or -1 if the text jpayne@69: * is NUL terminated. jpayne@69: * @param status any error conditions are reported back in this variable. jpayne@69: * jpayne@69: * @stable ICU 3.6 jpayne@69: */ jpayne@69: U_STABLE void U_EXPORT2 jpayne@69: ucsdet_setText(UCharsetDetector *ucsd, const char *textIn, int32_t len, UErrorCode *status); jpayne@69: jpayne@69: jpayne@69: /** Set the declared encoding for charset detection. jpayne@69: * The declared encoding of an input text is an encoding obtained jpayne@69: * by the user from an http header or xml declaration or similar source that jpayne@69: * can be provided as an additional hint to the charset detector. jpayne@69: * jpayne@69: * How and whether the declared encoding will be used during the jpayne@69: * detection process is TBD. jpayne@69: * jpayne@69: * @param ucsd the charset detector to be used. jpayne@69: * @param encoding an encoding for the current data obtained from jpayne@69: * a header or declaration or other source outside jpayne@69: * of the byte data itself. jpayne@69: * @param length the length of the encoding name, or -1 if the name string jpayne@69: * is NUL terminated. jpayne@69: * @param status any error conditions are reported back in this variable. jpayne@69: * jpayne@69: * @stable ICU 3.6 jpayne@69: */ jpayne@69: U_STABLE void U_EXPORT2 jpayne@69: ucsdet_setDeclaredEncoding(UCharsetDetector *ucsd, const char *encoding, int32_t length, UErrorCode *status); jpayne@69: jpayne@69: jpayne@69: /** jpayne@69: * Return the charset that best matches the supplied input data. jpayne@69: * jpayne@69: * Note though, that because the detection jpayne@69: * only looks at the start of the input data, jpayne@69: * there is a possibility that the returned charset will fail to handle jpayne@69: * the full set of input data. jpayne@69: *

jpayne@69: * The returned UCharsetMatch object is owned by the UCharsetDetector. jpayne@69: * It will remain valid until the detector input is reset, or until jpayne@69: * the detector is closed. jpayne@69: *

jpayne@69: * The function will fail if jpayne@69: *

jpayne@69: * jpayne@69: * @param ucsd the charset detector to be used. jpayne@69: * @param status any error conditions are reported back in this variable. jpayne@69: * @return a UCharsetMatch representing the best matching charset, jpayne@69: * or NULL if no charset matches the byte data. jpayne@69: * jpayne@69: * @stable ICU 3.6 jpayne@69: */ jpayne@69: U_STABLE const UCharsetMatch * U_EXPORT2 jpayne@69: ucsdet_detect(UCharsetDetector *ucsd, UErrorCode *status); jpayne@69: jpayne@69: jpayne@69: /** jpayne@69: * Find all charset matches that appear to be consistent with the input, jpayne@69: * returning an array of results. The results are ordered with the jpayne@69: * best quality match first. jpayne@69: * jpayne@69: * Because the detection only looks at a limited amount of the jpayne@69: * input byte data, some of the returned charsets may fail to handle jpayne@69: * the all of input data. jpayne@69: *

jpayne@69: * The returned UCharsetMatch objects are owned by the UCharsetDetector. jpayne@69: * They will remain valid until the detector is closed or modified jpayne@69: * jpayne@69: *

jpayne@69: * Return an error if jpayne@69: *

jpayne@69: * jpayne@69: * @param ucsd the charset detector to be used. jpayne@69: * @param matchesFound pointer to a variable that will be set to the jpayne@69: * number of charsets identified that are consistent with jpayne@69: * the input data. Output only. jpayne@69: * @param status any error conditions are reported back in this variable. jpayne@69: * @return A pointer to an array of pointers to UCharSetMatch objects. jpayne@69: * This array, and the UCharSetMatch instances to which it refers, jpayne@69: * are owned by the UCharsetDetector, and will remain valid until jpayne@69: * the detector is closed or modified. jpayne@69: * @stable ICU 3.6 jpayne@69: */ jpayne@69: U_STABLE const UCharsetMatch ** U_EXPORT2 jpayne@69: ucsdet_detectAll(UCharsetDetector *ucsd, int32_t *matchesFound, UErrorCode *status); jpayne@69: jpayne@69: jpayne@69: jpayne@69: /** jpayne@69: * Get the name of the charset represented by a UCharsetMatch. jpayne@69: * jpayne@69: * The storage for the returned name string is owned by the jpayne@69: * UCharsetMatch, and will remain valid while the UCharsetMatch jpayne@69: * is valid. jpayne@69: * jpayne@69: * The name returned is suitable for use with the ICU conversion APIs. jpayne@69: * jpayne@69: * @param ucsm The charset match object. jpayne@69: * @param status Any error conditions are reported back in this variable. jpayne@69: * @return The name of the matching charset. jpayne@69: * jpayne@69: * @stable ICU 3.6 jpayne@69: */ jpayne@69: U_STABLE const char * U_EXPORT2 jpayne@69: ucsdet_getName(const UCharsetMatch *ucsm, UErrorCode *status); jpayne@69: jpayne@69: /** jpayne@69: * Get a confidence number for the quality of the match of the byte jpayne@69: * data with the charset. Confidence numbers range from zero to 100, jpayne@69: * with 100 representing complete confidence and zero representing jpayne@69: * no confidence. jpayne@69: * jpayne@69: * The confidence values are somewhat arbitrary. They define an jpayne@69: * an ordering within the results for any single detection operation jpayne@69: * but are not generally comparable between the results for different input. jpayne@69: * jpayne@69: * A confidence value of ten does have a general meaning - it is used jpayne@69: * for charsets that can represent the input data, but for which there jpayne@69: * is no other indication that suggests that the charset is the correct one. jpayne@69: * Pure 7 bit ASCII data, for example, is compatible with a jpayne@69: * great many charsets, most of which will appear as possible matches jpayne@69: * with a confidence of 10. jpayne@69: * jpayne@69: * @param ucsm The charset match object. jpayne@69: * @param status Any error conditions are reported back in this variable. jpayne@69: * @return A confidence number for the charset match. jpayne@69: * jpayne@69: * @stable ICU 3.6 jpayne@69: */ jpayne@69: U_STABLE int32_t U_EXPORT2 jpayne@69: ucsdet_getConfidence(const UCharsetMatch *ucsm, UErrorCode *status); jpayne@69: jpayne@69: /** jpayne@69: * Get the RFC 3066 code for the language of the input data. jpayne@69: * jpayne@69: * The Charset Detection service is intended primarily for detecting jpayne@69: * charsets, not language. For some, but not all, charsets, a language is jpayne@69: * identified as a byproduct of the detection process, and that is what jpayne@69: * is returned by this function. jpayne@69: * jpayne@69: * CAUTION: jpayne@69: * 1. Language information is not available for input data encoded in jpayne@69: * all charsets. In particular, no language is identified jpayne@69: * for UTF-8 input data. jpayne@69: * jpayne@69: * 2. Closely related languages may sometimes be confused. jpayne@69: * jpayne@69: * If more accurate language detection is required, a linguistic jpayne@69: * analysis package should be used. jpayne@69: * jpayne@69: * The storage for the returned name string is owned by the jpayne@69: * UCharsetMatch, and will remain valid while the UCharsetMatch jpayne@69: * is valid. jpayne@69: * jpayne@69: * @param ucsm The charset match object. jpayne@69: * @param status Any error conditions are reported back in this variable. jpayne@69: * @return The RFC 3066 code for the language of the input data, or jpayne@69: * an empty string if the language could not be determined. jpayne@69: * jpayne@69: * @stable ICU 3.6 jpayne@69: */ jpayne@69: U_STABLE const char * U_EXPORT2 jpayne@69: ucsdet_getLanguage(const UCharsetMatch *ucsm, UErrorCode *status); jpayne@69: jpayne@69: jpayne@69: /** jpayne@69: * Get the entire input text as a UChar string, placing it into jpayne@69: * a caller-supplied buffer. A terminating jpayne@69: * NUL character will be appended to the buffer if space is available. jpayne@69: * jpayne@69: * The number of UChars in the output string, not including the terminating jpayne@69: * NUL, is returned. jpayne@69: * jpayne@69: * If the supplied buffer is smaller than required to hold the output, jpayne@69: * the contents of the buffer are undefined. The full output string length jpayne@69: * (in UChars) is returned as always, and can be used to allocate a buffer jpayne@69: * of the correct size. jpayne@69: * jpayne@69: * jpayne@69: * @param ucsm The charset match object. jpayne@69: * @param buf A UChar buffer to be filled with the converted text data. jpayne@69: * @param cap The capacity of the buffer in UChars. jpayne@69: * @param status Any error conditions are reported back in this variable. jpayne@69: * @return The number of UChars in the output string. jpayne@69: * jpayne@69: * @stable ICU 3.6 jpayne@69: */ jpayne@69: U_STABLE int32_t U_EXPORT2 jpayne@69: ucsdet_getUChars(const UCharsetMatch *ucsm, jpayne@69: UChar *buf, int32_t cap, UErrorCode *status); jpayne@69: jpayne@69: jpayne@69: jpayne@69: /** jpayne@69: * Get an iterator over the set of all detectable charsets - jpayne@69: * over the charsets that are known to the charset detection jpayne@69: * service. jpayne@69: * jpayne@69: * The returned UEnumeration provides access to the names of jpayne@69: * the charsets. jpayne@69: * jpayne@69: *

jpayne@69: * The state of the Charset detector that is passed in does not jpayne@69: * affect the result of this function, but requiring a valid, open jpayne@69: * charset detector as a parameter insures that the charset detection jpayne@69: * service has been safely initialized and that the required detection jpayne@69: * data is available. jpayne@69: * jpayne@69: *

jpayne@69: * Note: Multiple different charset encodings in a same family may use jpayne@69: * a single shared name in this implementation. For example, this method returns jpayne@69: * an array including "ISO-8859-1" (ISO Latin 1), but not including "windows-1252" jpayne@69: * (Windows Latin 1). However, actual detection result could be "windows-1252" jpayne@69: * when the input data matches Latin 1 code points with any points only available jpayne@69: * in "windows-1252". jpayne@69: * jpayne@69: * @param ucsd a Charset detector. jpayne@69: * @param status Any error conditions are reported back in this variable. jpayne@69: * @return an iterator providing access to the detectable charset names. jpayne@69: * @stable ICU 3.6 jpayne@69: */ jpayne@69: U_STABLE UEnumeration * U_EXPORT2 jpayne@69: ucsdet_getAllDetectableCharsets(const UCharsetDetector *ucsd, UErrorCode *status); jpayne@69: jpayne@69: /** jpayne@69: * Test whether input filtering is enabled for this charset detector. jpayne@69: * Input filtering removes text that appears to be HTML or xml jpayne@69: * markup from the input before applying the code page detection jpayne@69: * heuristics. jpayne@69: * jpayne@69: * @param ucsd The charset detector to check. jpayne@69: * @return TRUE if filtering is enabled. jpayne@69: * @stable ICU 3.6 jpayne@69: */ jpayne@69: jpayne@69: U_STABLE UBool U_EXPORT2 jpayne@69: ucsdet_isInputFilterEnabled(const UCharsetDetector *ucsd); jpayne@69: jpayne@69: jpayne@69: /** jpayne@69: * Enable filtering of input text. If filtering is enabled, jpayne@69: * text within angle brackets ("<" and ">") will be removed jpayne@69: * before detection, which will remove most HTML or xml markup. jpayne@69: * jpayne@69: * @param ucsd the charset detector to be modified. jpayne@69: * @param filter true to enable input text filtering. jpayne@69: * @return The previous setting. jpayne@69: * jpayne@69: * @stable ICU 3.6 jpayne@69: */ jpayne@69: U_STABLE UBool U_EXPORT2 jpayne@69: ucsdet_enableInputFilter(UCharsetDetector *ucsd, UBool filter); jpayne@69: jpayne@69: #ifndef U_HIDE_INTERNAL_API jpayne@69: /** jpayne@69: * Get an iterator over the set of detectable charsets - jpayne@69: * over the charsets that are enabled by the specified charset detector. jpayne@69: * jpayne@69: * The returned UEnumeration provides access to the names of jpayne@69: * the charsets. jpayne@69: * jpayne@69: * @param ucsd a Charset detector. jpayne@69: * @param status Any error conditions are reported back in this variable. jpayne@69: * @return an iterator providing access to the detectable charset names by jpayne@69: * the specified charset detector. jpayne@69: * @internal jpayne@69: */ jpayne@69: U_INTERNAL UEnumeration * U_EXPORT2 jpayne@69: ucsdet_getDetectableCharsets(const UCharsetDetector *ucsd, UErrorCode *status); jpayne@69: jpayne@69: /** jpayne@69: * Enable or disable individual charset encoding. jpayne@69: * A name of charset encoding must be included in the names returned by jpayne@69: * {@link #ucsdet_getAllDetectableCharsets()}. jpayne@69: * jpayne@69: * @param ucsd a Charset detector. jpayne@69: * @param encoding encoding the name of charset encoding. jpayne@69: * @param enabled TRUE to enable, or FALSE to disable the jpayne@69: * charset encoding. jpayne@69: * @param status receives the return status. When the name of charset encoding jpayne@69: * is not supported, U_ILLEGAL_ARGUMENT_ERROR is set. jpayne@69: * @internal jpayne@69: */ jpayne@69: U_INTERNAL void U_EXPORT2 jpayne@69: ucsdet_setDetectableCharset(UCharsetDetector *ucsd, const char *encoding, UBool enabled, UErrorCode *status); jpayne@69: #endif /* U_HIDE_INTERNAL_API */ jpayne@69: jpayne@69: #endif jpayne@69: #endif /* __UCSDET_H */ jpayne@69: jpayne@69: