diff options
Diffstat (limited to 'deps/node/deps/icu-small/source/common/ucnvmbcs.cpp')
| -rw-r--r-- | deps/node/deps/icu-small/source/common/ucnvmbcs.cpp | 5723 |
1 files changed, 0 insertions, 5723 deletions
diff --git a/deps/node/deps/icu-small/source/common/ucnvmbcs.cpp b/deps/node/deps/icu-small/source/common/ucnvmbcs.cpp deleted file mode 100644 index e1248a7b..00000000 --- a/deps/node/deps/icu-small/source/common/ucnvmbcs.cpp +++ /dev/null @@ -1,5723 +0,0 @@ -// © 2016 and later: Unicode, Inc. and others. -// License & terms of use: http://www.unicode.org/copyright.html -/* -****************************************************************************** -* -* Copyright (C) 2000-2016, International Business Machines -* Corporation and others. All Rights Reserved. -* -****************************************************************************** -* file name: ucnvmbcs.cpp -* encoding: UTF-8 -* tab size: 8 (not used) -* indentation:4 -* -* created on: 2000jul03 -* created by: Markus W. Scherer -* -* The current code in this file replaces the previous implementation -* of conversion code from multi-byte codepages to Unicode and back. -* This implementation supports the following: -* - legacy variable-length codepages with up to 4 bytes per character -* - all Unicode code points (up to 0x10ffff) -* - efficient distinction of unassigned vs. illegal byte sequences -* - it is possible in fromUnicode() to directly deal with simple -* stateful encodings (used for EBCDIC_STATEFUL) -* - it is possible to convert Unicode code points -* to a single zero byte (but not as a fallback except for SBCS) -* -* Remaining limitations in fromUnicode: -* - byte sequences must not have leading zero bytes -* - except for SBCS codepages: no fallback mapping from Unicode to a zero byte -* - limitation to up to 4 bytes per character -* -* ICU 2.8 (late 2003) adds a secondary data structure which lifts some of these -* limitations and adds m:n character mappings and other features. -* See ucnv_ext.h for details. -* -* Change history: -* -* 5/6/2001 Ram Moved MBCS_SINGLE_RESULT_FROM_U,MBCS_STAGE_2_FROM_U, -* MBCS_VALUE_2_FROM_STAGE_2, MBCS_VALUE_4_FROM_STAGE_2 -* macros to ucnvmbcs.h file -*/ - -#include "unicode/utypes.h" - -#if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION - -#include "unicode/ucnv.h" -#include "unicode/ucnv_cb.h" -#include "unicode/udata.h" -#include "unicode/uset.h" -#include "unicode/utf8.h" -#include "unicode/utf16.h" -#include "ucnv_bld.h" -#include "ucnvmbcs.h" -#include "ucnv_ext.h" -#include "ucnv_cnv.h" -#include "cmemory.h" -#include "cstring.h" -#include "umutex.h" -#include "ustr_imp.h" - -/* control optimizations according to the platform */ -#define MBCS_UNROLL_SINGLE_TO_BMP 1 -#define MBCS_UNROLL_SINGLE_FROM_BMP 0 - -/* - * _MBCSHeader versions 5.3 & 4.3 - * (Note that the _MBCSHeader version is in addition to the converter formatVersion.) - * - * This version is optional. Version 5 is used for incompatible data format changes. - * makeconv will continue to generate version 4 files if possible. - * - * Changes from version 4: - * - * The main difference is an additional _MBCSHeader field with - * - the length (number of uint32_t) of the _MBCSHeader - * - flags for further incompatible data format changes - * - flags for further, backward compatible data format changes - * - * The MBCS_OPT_FROM_U flag indicates that most of the fromUnicode data is omitted from - * the file and needs to be reconstituted at load time. - * This requires a utf8Friendly format with an additional mbcsIndex table for fast - * (and UTF-8-friendly) fromUnicode conversion for Unicode code points up to maxFastUChar. - * (For details about these structures see below, and see ucnvmbcs.h.) - * - * utf8Friendly also implies that the fromUnicode mappings are stored in ascending order - * of the Unicode code points. (This requires that the .ucm file has the |0 etc. - * precision markers for all mappings.) - * - * All fallbacks have been moved to the extension table, leaving only roundtrips in the - * omitted data that can be reconstituted from the toUnicode data. - * - * Of the stage 2 table, the part corresponding to maxFastUChar and below is omitted. - * With only roundtrip mappings in the base fromUnicode data, this part is fully - * redundant with the mbcsIndex and will be reconstituted from that (also using the - * stage 1 table which contains the information about how stage 2 was compacted). - * - * The rest of the stage 2 table, the part for code points above maxFastUChar, - * is stored in the file and will be appended to the reconstituted part. - * - * The entire fromUBytes array is omitted from the file and will be reconstitued. - * This is done by enumerating all toUnicode roundtrip mappings, performing - * each mapping (using the stage 1 and reconstituted stage 2 tables) and - * writing instead of reading the byte values. - * - * _MBCSHeader version 4.3 - * - * Change from version 4.2: - * - Optional utf8Friendly data structures, with 64-entry stage 3 block - * allocation for parts of the BMP, and an additional mbcsIndex in non-SBCS - * files which can be used instead of stages 1 & 2. - * Faster lookups for roundtrips from most commonly used characters, - * and lookups from UTF-8 byte sequences with a natural bit distribution. - * See ucnvmbcs.h for more details. - * - * Change from version 4.1: - * - Added an optional extension table structure at the end of the .cnv file. - * It is present if the upper bits of the header flags field contains a non-zero - * byte offset to it. - * Files that contain only a conversion table and no base table - * use the special outputType MBCS_OUTPUT_EXT_ONLY. - * These contain the base table name between the MBCS header and the extension - * data. - * - * Change from version 4.0: - * - Replace header.reserved with header.fromUBytesLength so that all - * fields in the data have length. - * - * Changes from version 3 (for performance improvements): - * - new bit distribution for state table entries - * - reordered action codes - * - new data structure for single-byte fromUnicode - * + stage 2 only contains indexes - * + stage 3 stores 16 bits per character with classification bits 15..8 - * - no multiplier for stage 1 entries - * - stage 2 for non-single-byte codepages contains the index and the flags in - * one 32-bit value - * - 2-byte and 4-byte fromUnicode results are stored directly as 16/32-bit integers - * - * For more details about old versions of the MBCS data structure, see - * the corresponding versions of this file. - * - * Converting stateless codepage data ---------------------------------------*** - * (or codepage data with simple states) to Unicode. - * - * Data structure and algorithm for converting from complex legacy codepages - * to Unicode. (Designed before 2000-may-22.) - * - * The basic idea is that the structure of legacy codepages can be described - * with state tables. - * When reading a byte stream, each input byte causes a state transition. - * Some transitions result in the output of a code point, some result in - * "unassigned" or "illegal" output. - * This is used here for character conversion. - * - * The data structure begins with a state table consisting of a row - * per state, with 256 entries (columns) per row for each possible input - * byte value. - * Each entry is 32 bits wide, with two formats distinguished by - * the sign bit (bit 31): - * - * One format for transitional entries (bit 31 not set) for non-final bytes, and - * one format for final entries (bit 31 set). - * Both formats contain the number of the next state in the same bit - * positions. - * State 0 is the initial state. - * - * Most of the time, the offset values of subsequent states are added - * up to a scalar value. This value will eventually be the index of - * the Unicode code point in a table that follows the state table. - * The effect is that the code points for final state table rows - * are contiguous. The code points of final state rows follow each other - * in the order of the references to those final states by previous - * states, etc. - * - * For some terminal states, the offset is itself the output Unicode - * code point (16 bits for a BMP code point or 20 bits for a supplementary - * code point (stored as code point minus 0x10000 so that 20 bits are enough). - * For others, the code point in the Unicode table is stored with either - * one or two code units: one for BMP code points, two for a pair of - * surrogates. - * All code points for a final state entry take up the same number of code - * units, regardless of whether they all actually _use_ the same number - * of code units. This is necessary for simple array access. - * - * An additional feature comes in with what in ICU is called "fallback" - * mappings: - * - * In addition to round-trippable, precise, 1:1 mappings, there are often - * mappings defined between similar, though not the same, characters. - * Typically, such mappings occur only in fromUnicode mapping tables because - * Unicode has a superset repertoire of most other codepages. However, it - * is possible to provide such mappings in the toUnicode tables, too. - * In this case, the fallback mappings are partly integrated into the - * general state tables because the structure of the encoding includes their - * byte sequences. - * For final entries in an initial state, fallback mappings are stored in - * the entry itself like with roundtrip mappings. - * For other final entries, they are stored in the code units table if - * the entry is for a pair of code units. - * For single-unit results in the code units table, there is no space to - * alternatively hold a fallback mapping; in this case, the code unit - * is stored as U+fffe (unassigned), and the fallback mapping needs to - * be looked up by the scalar offset value in a separate table. - * - * "Unassigned" state entries really mean "structurally unassigned", - * i.e., such a byte sequence will never have a mapping result. - * - * The interpretation of the bits in each entry is as follows: - * - * Bit 31 not set, not a terminal entry ("transitional"): - * 30..24 next state - * 23..0 offset delta, to be added up - * - * Bit 31 set, terminal ("final") entry: - * 30..24 next state (regardless of action code) - * 23..20 action code: - * action codes 0 and 1 result in precise-mapping Unicode code points - * 0 valid byte sequence - * 19..16 not used, 0 - * 15..0 16-bit Unicode BMP code point - * never U+fffe or U+ffff - * 1 valid byte sequence - * 19..0 20-bit Unicode supplementary code point - * never U+fffe or U+ffff - * - * action codes 2 and 3 result in fallback (unidirectional-mapping) Unicode code points - * 2 valid byte sequence (fallback) - * 19..16 not used, 0 - * 15..0 16-bit Unicode BMP code point as fallback result - * 3 valid byte sequence (fallback) - * 19..0 20-bit Unicode supplementary code point as fallback result - * - * action codes 4 and 5 may result in roundtrip/fallback/unassigned/illegal results - * depending on the code units they result in - * 4 valid byte sequence - * 19..9 not used, 0 - * 8..0 final offset delta - * pointing to one 16-bit code unit which may be - * fffe unassigned -- look for a fallback for this offset - * ffff illegal - * 5 valid byte sequence - * 19..9 not used, 0 - * 8..0 final offset delta - * pointing to two 16-bit code units - * (typically UTF-16 surrogates) - * the result depends on the first code unit as follows: - * 0000..d7ff roundtrip BMP code point (1st alone) - * d800..dbff roundtrip surrogate pair (1st, 2nd) - * dc00..dfff fallback surrogate pair (1st-400, 2nd) - * e000 roundtrip BMP code point (2nd alone) - * e001 fallback BMP code point (2nd alone) - * fffe unassigned - * ffff illegal - * (the final offset deltas are at most 255 * 2, - * times 2 because of storing code unit pairs) - * - * 6 unassigned byte sequence - * 19..16 not used, 0 - * 15..0 16-bit Unicode BMP code point U+fffe (new with version 2) - * this does not contain a final offset delta because the main - * purpose of this action code is to save scalar offset values; - * therefore, fallback values cannot be assigned to byte - * sequences that result in this action code - * 7 illegal byte sequence - * 19..16 not used, 0 - * 15..0 16-bit Unicode BMP code point U+ffff (new with version 2) - * 8 state change only - * 19..0 not used, 0 - * useful for state changes in simple stateful encodings, - * at Shift-In/Shift-Out codes - * - * - * 9..15 reserved for future use - * current implementations will only perform a state change - * and ignore bits 19..0 - * - * An encoding with contiguous ranges of unassigned byte sequences, like - * Shift-JIS and especially EUC-TW, can be stored efficiently by having - * at least two states for the trail bytes: - * One trail byte state that results in code points, and one that only - * has "unassigned" and "illegal" terminal states. - * - * Note: partly by accident, this data structure supports simple stateful - * encodings without any additional logic. - * Currently, only simple Shift-In/Shift-Out schemes are handled with - * appropriate state tables (especially EBCDIC_STATEFUL!). - * - * MBCS version 2 added: - * unassigned and illegal action codes have U+fffe and U+ffff - * instead of unused bits; this is useful for _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP() - * - * Converting from Unicode to codepage bytes --------------------------------*** - * - * The conversion data structure for fromUnicode is designed for the known - * structure of Unicode. It maps from 21-bit code points (0..0x10ffff) to - * a sequence of 1..4 bytes, in addition to a flag that indicates if there is - * a roundtrip mapping. - * - * The lookup is done with a 3-stage trie, using 11/6/4 bits for stage 1/2/3 - * like in the character properties table. - * The beginning of the trie is at offsetFromUTable, the beginning of stage 3 - * with the resulting bytes is at offsetFromUBytes. - * - * Beginning with version 4, single-byte codepages have a significantly different - * trie compared to other codepages. - * In all cases, the entry in stage 1 is directly the index of the block of - * 64 entries in stage 2. - * - * Single-byte lookup: - * - * Stage 2 only contains 16-bit indexes directly to the 16-blocks in stage 3. - * Stage 3 contains one 16-bit word per result: - * Bits 15..8 indicate the kind of result: - * f roundtrip result - * c fallback result from private-use code point - * 8 fallback result from other code points - * 0 unassigned - * Bits 7..0 contain the codepage byte. A zero byte is always possible. - * - * In version 4.3, the runtime code can build an sbcsIndex for a utf8Friendly - * file. For 2-byte UTF-8 byte sequences and some 3-byte sequences the lookup - * becomes a 2-stage (single-index) trie lookup with 6 bits for stage 3. - * ASCII code points can be looked up with a linear array access into stage 3. - * See maxFastUChar and other details in ucnvmbcs.h. - * - * Multi-byte lookup: - * - * Stage 2 contains a 32-bit word for each 16-block in stage 3: - * Bits 31..16 contain flags for which stage 3 entries contain roundtrip results - * test: MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) - * If this test is false, then a non-zero result will be interpreted as - * a fallback mapping. - * Bits 15..0 contain the index to stage 3, which must be multiplied by 16*(bytes per char) - * - * Stage 3 contains 2, 3, or 4 bytes per result. - * 2 or 4 bytes are stored as uint16_t/uint32_t in platform endianness, - * while 3 bytes are stored as bytes in big-endian order. - * Leading zero bytes are ignored, and the number of bytes is counted. - * A zero byte mapping result is possible as a roundtrip result. - * For some output types, the actual result is processed from this; - * see ucnv_MBCSFromUnicodeWithOffsets(). - * - * Note that stage 1 always contains 0x440=1088 entries (0x440==0x110000>>10), - * or (version 3 and up) for BMP-only codepages, it contains 64 entries. - * - * In version 4.3, a utf8Friendly file contains an mbcsIndex table. - * For 2-byte UTF-8 byte sequences and most 3-byte sequences the lookup - * becomes a 2-stage (single-index) trie lookup with 6 bits for stage 3. - * ASCII code points can be looked up with a linear array access into stage 3. - * See maxFastUChar, mbcsIndex and other details in ucnvmbcs.h. - * - * In version 3, stage 2 blocks may overlap by multiples of the multiplier - * for compaction. - * In version 4, stage 2 blocks (and for single-byte codepages, stage 3 blocks) - * may overlap by any number of entries. - * - * MBCS version 2 added: - * the converter checks for known output types, which allows - * adding new ones without crashing an unaware converter - */ - -/** - * Callback from ucnv_MBCSEnumToUnicode(), takes 32 mappings from - * consecutive sequences of bytes, starting from the one encoded in value, - * to Unicode code points. (Multiple mappings to reduce per-function call overhead.) - * Does not currently support m:n mappings or reverse fallbacks. - * This function will not be called for sequences of bytes with leading zeros. - * - * @param context an opaque pointer, as passed into ucnv_MBCSEnumToUnicode() - * @param value contains 1..4 bytes of the first byte sequence, right-aligned - * @param codePoints resulting Unicode code points, or negative if a byte sequence does - * not map to anything - * @return TRUE to continue enumeration, FALSE to stop - */ -typedef UBool U_CALLCONV -UConverterEnumToUCallback(const void *context, uint32_t value, UChar32 codePoints[32]); - -static void U_CALLCONV -ucnv_MBCSLoad(UConverterSharedData *sharedData, - UConverterLoadArgs *pArgs, - const uint8_t *raw, - UErrorCode *pErrorCode); - -static void U_CALLCONV -ucnv_MBCSUnload(UConverterSharedData *sharedData); - -static void U_CALLCONV -ucnv_MBCSOpen(UConverter *cnv, - UConverterLoadArgs *pArgs, - UErrorCode *pErrorCode); - -static UChar32 U_CALLCONV -ucnv_MBCSGetNextUChar(UConverterToUnicodeArgs *pArgs, - UErrorCode *pErrorCode); - -static void U_CALLCONV -ucnv_MBCSGetStarters(const UConverter* cnv, - UBool starters[256], - UErrorCode *pErrorCode); - -U_CDECL_BEGIN -static const char* U_CALLCONV -ucnv_MBCSGetName(const UConverter *cnv); -U_CDECL_END - -static void U_CALLCONV -ucnv_MBCSWriteSub(UConverterFromUnicodeArgs *pArgs, - int32_t offsetIndex, - UErrorCode *pErrorCode); - -static UChar32 U_CALLCONV -ucnv_MBCSGetNextUChar(UConverterToUnicodeArgs *pArgs, - UErrorCode *pErrorCode); - -static void U_CALLCONV -ucnv_SBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs, - UConverterToUnicodeArgs *pToUArgs, - UErrorCode *pErrorCode); - -static void U_CALLCONV -ucnv_MBCSGetUnicodeSet(const UConverter *cnv, - const USetAdder *sa, - UConverterUnicodeSet which, - UErrorCode *pErrorCode); - -static void U_CALLCONV -ucnv_DBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs, - UConverterToUnicodeArgs *pToUArgs, - UErrorCode *pErrorCode); - -static const UConverterImpl _SBCSUTF8Impl={ - UCNV_MBCS, - - ucnv_MBCSLoad, - ucnv_MBCSUnload, - - ucnv_MBCSOpen, - NULL, - NULL, - - ucnv_MBCSToUnicodeWithOffsets, - ucnv_MBCSToUnicodeWithOffsets, - ucnv_MBCSFromUnicodeWithOffsets, - ucnv_MBCSFromUnicodeWithOffsets, - ucnv_MBCSGetNextUChar, - - ucnv_MBCSGetStarters, - ucnv_MBCSGetName, - ucnv_MBCSWriteSub, - NULL, - ucnv_MBCSGetUnicodeSet, - - NULL, - ucnv_SBCSFromUTF8 -}; - -static const UConverterImpl _DBCSUTF8Impl={ - UCNV_MBCS, - - ucnv_MBCSLoad, - ucnv_MBCSUnload, - - ucnv_MBCSOpen, - NULL, - NULL, - - ucnv_MBCSToUnicodeWithOffsets, - ucnv_MBCSToUnicodeWithOffsets, - ucnv_MBCSFromUnicodeWithOffsets, - ucnv_MBCSFromUnicodeWithOffsets, - ucnv_MBCSGetNextUChar, - - ucnv_MBCSGetStarters, - ucnv_MBCSGetName, - ucnv_MBCSWriteSub, - NULL, - ucnv_MBCSGetUnicodeSet, - - NULL, - ucnv_DBCSFromUTF8 -}; - -static const UConverterImpl _MBCSImpl={ - UCNV_MBCS, - - ucnv_MBCSLoad, - ucnv_MBCSUnload, - - ucnv_MBCSOpen, - NULL, - NULL, - - ucnv_MBCSToUnicodeWithOffsets, - ucnv_MBCSToUnicodeWithOffsets, - ucnv_MBCSFromUnicodeWithOffsets, - ucnv_MBCSFromUnicodeWithOffsets, - ucnv_MBCSGetNextUChar, - - ucnv_MBCSGetStarters, - ucnv_MBCSGetName, - ucnv_MBCSWriteSub, - NULL, - ucnv_MBCSGetUnicodeSet, - NULL, - NULL -}; - -/* Static data is in tools/makeconv/ucnvstat.c for data-based - * converters. Be sure to update it as well. - */ - -const UConverterSharedData _MBCSData={ - sizeof(UConverterSharedData), 1, - NULL, NULL, FALSE, TRUE, &_MBCSImpl, - 0, UCNV_MBCS_TABLE_INITIALIZER -}; - - -/* GB 18030 data ------------------------------------------------------------ */ - -/* helper macros for linear values for GB 18030 four-byte sequences */ -#define LINEAR_18030(a, b, c, d) ((((a)*10+(b))*126L+(c))*10L+(d)) - -#define LINEAR_18030_BASE LINEAR_18030(0x81, 0x30, 0x81, 0x30) - -#define LINEAR(x) LINEAR_18030(x>>24, (x>>16)&0xff, (x>>8)&0xff, x&0xff) - -/* - * Some ranges of GB 18030 where both the Unicode code points and the - * GB four-byte sequences are contiguous and are handled algorithmically by - * the special callback functions below. - * The values are start & end of Unicode & GB codes. - * - * Note that single surrogates are not mapped by GB 18030 - * as of the re-released mapping tables from 2000-nov-30. - */ -static const uint32_t -gb18030Ranges[14][4]={ - {0x10000, 0x10FFFF, LINEAR(0x90308130), LINEAR(0xE3329A35)}, - {0x9FA6, 0xD7FF, LINEAR(0x82358F33), LINEAR(0x8336C738)}, - {0x0452, 0x1E3E, LINEAR(0x8130D330), LINEAR(0x8135F436)}, - {0x1E40, 0x200F, LINEAR(0x8135F438), LINEAR(0x8136A531)}, - {0xE865, 0xF92B, LINEAR(0x8336D030), LINEAR(0x84308534)}, - {0x2643, 0x2E80, LINEAR(0x8137A839), LINEAR(0x8138FD38)}, - {0xFA2A, 0xFE2F, LINEAR(0x84309C38), LINEAR(0x84318537)}, - {0x3CE1, 0x4055, LINEAR(0x8231D438), LINEAR(0x8232AF32)}, - {0x361B, 0x3917, LINEAR(0x8230A633), LINEAR(0x8230F237)}, - {0x49B8, 0x4C76, LINEAR(0x8234A131), LINEAR(0x8234E733)}, - {0x4160, 0x4336, LINEAR(0x8232C937), LINEAR(0x8232F837)}, - {0x478E, 0x4946, LINEAR(0x8233E838), LINEAR(0x82349638)}, - {0x44D7, 0x464B, LINEAR(0x8233A339), LINEAR(0x8233C931)}, - {0xFFE6, 0xFFFF, LINEAR(0x8431A234), LINEAR(0x8431A439)} -}; - -/* bit flag for UConverter.options indicating GB 18030 special handling */ -#define _MBCS_OPTION_GB18030 0x8000 - -/* bit flag for UConverter.options indicating KEIS,JEF,JIF special handling */ -#define _MBCS_OPTION_KEIS 0x01000 -#define _MBCS_OPTION_JEF 0x02000 -#define _MBCS_OPTION_JIPS 0x04000 - -#define KEIS_SO_CHAR_1 0x0A -#define KEIS_SO_CHAR_2 0x42 -#define KEIS_SI_CHAR_1 0x0A -#define KEIS_SI_CHAR_2 0x41 - -#define JEF_SO_CHAR 0x28 -#define JEF_SI_CHAR 0x29 - -#define JIPS_SO_CHAR_1 0x1A -#define JIPS_SO_CHAR_2 0x70 -#define JIPS_SI_CHAR_1 0x1A -#define JIPS_SI_CHAR_2 0x71 - -enum SISO_Option { - SI, - SO -}; -typedef enum SISO_Option SISO_Option; - -static int32_t getSISOBytes(SISO_Option option, uint32_t cnvOption, uint8_t *value) { - int32_t SISOLength = 0; - - switch (option) { - case SI: - if ((cnvOption&_MBCS_OPTION_KEIS)!=0) { - value[0] = KEIS_SI_CHAR_1; - value[1] = KEIS_SI_CHAR_2; - SISOLength = 2; - } else if ((cnvOption&_MBCS_OPTION_JEF)!=0) { - value[0] = JEF_SI_CHAR; - SISOLength = 1; - } else if ((cnvOption&_MBCS_OPTION_JIPS)!=0) { - value[0] = JIPS_SI_CHAR_1; - value[1] = JIPS_SI_CHAR_2; - SISOLength = 2; - } else { - value[0] = UCNV_SI; - SISOLength = 1; - } - break; - case SO: - if ((cnvOption&_MBCS_OPTION_KEIS)!=0) { - value[0] = KEIS_SO_CHAR_1; - value[1] = KEIS_SO_CHAR_2; - SISOLength = 2; - } else if ((cnvOption&_MBCS_OPTION_JEF)!=0) { - value[0] = JEF_SO_CHAR; - SISOLength = 1; - } else if ((cnvOption&_MBCS_OPTION_JIPS)!=0) { - value[0] = JIPS_SO_CHAR_1; - value[1] = JIPS_SO_CHAR_2; - SISOLength = 2; - } else { - value[0] = UCNV_SO; - SISOLength = 1; - } - break; - default: - /* Should never happen. */ - break; - } - - return SISOLength; -} - -/* Miscellaneous ------------------------------------------------------------ */ - -/* similar to ucnv_MBCSGetNextUChar() but recursive */ -static UBool -enumToU(UConverterMBCSTable *mbcsTable, int8_t stateProps[], - int32_t state, uint32_t offset, - uint32_t value, - UConverterEnumToUCallback *callback, const void *context, - UErrorCode *pErrorCode) { - UChar32 codePoints[32]; - const int32_t *row; - const uint16_t *unicodeCodeUnits; - UChar32 anyCodePoints; - int32_t b, limit; - - row=mbcsTable->stateTable[state]; - unicodeCodeUnits=mbcsTable->unicodeCodeUnits; - - value<<=8; - anyCodePoints=-1; /* becomes non-negative if there is a mapping */ - - b=(stateProps[state]&0x38)<<2; - if(b==0 && stateProps[state]>=0x40) { - /* skip byte sequences with leading zeros because they are not stored in the fromUnicode table */ - codePoints[0]=U_SENTINEL; - b=1; - } - limit=((stateProps[state]&7)+1)<<5; - while(b<limit) { - int32_t entry=row[b]; - if(MBCS_ENTRY_IS_TRANSITION(entry)) { - int32_t nextState=MBCS_ENTRY_TRANSITION_STATE(entry); - if(stateProps[nextState]>=0) { - /* recurse to a state with non-ignorable actions */ - if(!enumToU( - mbcsTable, stateProps, nextState, - offset+MBCS_ENTRY_TRANSITION_OFFSET(entry), - value|(uint32_t)b, - callback, context, - pErrorCode)) { - return FALSE; - } - } - codePoints[b&0x1f]=U_SENTINEL; - } else { - UChar32 c; - int32_t action; - - /* - * An if-else-if chain provides more reliable performance for - * the most common cases compared to a switch. - */ - action=MBCS_ENTRY_FINAL_ACTION(entry); - if(action==MBCS_STATE_VALID_DIRECT_16) { - /* output BMP code point */ - c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - } else if(action==MBCS_STATE_VALID_16) { - int32_t finalOffset=offset+MBCS_ENTRY_FINAL_VALUE_16(entry); - c=unicodeCodeUnits[finalOffset]; - if(c<0xfffe) { - /* output BMP code point */ - } else { - c=U_SENTINEL; - } - } else if(action==MBCS_STATE_VALID_16_PAIR) { - int32_t finalOffset=offset+MBCS_ENTRY_FINAL_VALUE_16(entry); - c=unicodeCodeUnits[finalOffset++]; - if(c<0xd800) { - /* output BMP code point below 0xd800 */ - } else if(c<=0xdbff) { - /* output roundtrip or fallback supplementary code point */ - c=((c&0x3ff)<<10)+unicodeCodeUnits[finalOffset]+(0x10000-0xdc00); - } else if(c==0xe000) { - /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */ - c=unicodeCodeUnits[finalOffset]; - } else { - c=U_SENTINEL; - } - } else if(action==MBCS_STATE_VALID_DIRECT_20) { - /* output supplementary code point */ - c=(UChar32)(MBCS_ENTRY_FINAL_VALUE(entry)+0x10000); - } else { - c=U_SENTINEL; - } - - codePoints[b&0x1f]=c; - anyCodePoints&=c; - } - if(((++b)&0x1f)==0) { - if(anyCodePoints>=0) { - if(!callback(context, value|(uint32_t)(b-0x20), codePoints)) { - return FALSE; - } - anyCodePoints=-1; - } - } - } - return TRUE; -} - -/* - * Only called if stateProps[state]==-1. - * A recursive call may do stateProps[state]|=0x40 if this state is the target of an - * MBCS_STATE_CHANGE_ONLY. - */ -static int8_t -getStateProp(const int32_t (*stateTable)[256], int8_t stateProps[], int state) { - const int32_t *row; - int32_t min, max, entry, nextState; - - row=stateTable[state]; - stateProps[state]=0; - - /* find first non-ignorable state */ - for(min=0;; ++min) { - entry=row[min]; - nextState=MBCS_ENTRY_STATE(entry); - if(stateProps[nextState]==-1) { - getStateProp(stateTable, stateProps, nextState); - } - if(MBCS_ENTRY_IS_TRANSITION(entry)) { - if(stateProps[nextState]>=0) { - break; - } - } else if(MBCS_ENTRY_FINAL_ACTION(entry)<MBCS_STATE_UNASSIGNED) { - break; - } - if(min==0xff) { - stateProps[state]=-0x40; /* (int8_t)0xc0 */ - return stateProps[state]; - } - } - stateProps[state]|=(int8_t)((min>>5)<<3); - - /* find last non-ignorable state */ - for(max=0xff; min<max; --max) { - entry=row[max]; - nextState=MBCS_ENTRY_STATE(entry); - if(stateProps[nextState]==-1) { - getStateProp(stateTable, stateProps, nextState); - } - if(MBCS_ENTRY_IS_TRANSITION(entry)) { - if(stateProps[nextState]>=0) { - break; - } - } else if(MBCS_ENTRY_FINAL_ACTION(entry)<MBCS_STATE_UNASSIGNED) { - break; - } - } - stateProps[state]|=(int8_t)(max>>5); - - /* recurse further and collect direct-state information */ - while(min<=max) { - entry=row[min]; - nextState=MBCS_ENTRY_STATE(entry); - if(stateProps[nextState]==-1) { - getStateProp(stateTable, stateProps, nextState); - } - if(MBCS_ENTRY_IS_FINAL(entry)) { - stateProps[nextState]|=0x40; - if(MBCS_ENTRY_FINAL_ACTION(entry)<=MBCS_STATE_FALLBACK_DIRECT_20) { - stateProps[state]|=0x40; - } - } - ++min; - } - return stateProps[state]; -} - -/* - * Internal function enumerating the toUnicode data of an MBCS converter. - * Currently only used for reconstituting data for a MBCS_OPT_NO_FROM_U - * table, but could also be used for a future ucnv_getUnicodeSet() option - * that includes reverse fallbacks (after updating this function's implementation). - * Currently only handles roundtrip mappings. - * Does not currently handle extensions. - */ -static void -ucnv_MBCSEnumToUnicode(UConverterMBCSTable *mbcsTable, - UConverterEnumToUCallback *callback, const void *context, - UErrorCode *pErrorCode) { - /* - * Properties for each state, to speed up the enumeration. - * Ignorable actions are unassigned/illegal/state-change-only: - * They do not lead to mappings. - * - * Bits 7..6: - * 1 direct/initial state (stateful converters have multiple) - * 0 non-initial state with transitions or with non-ignorable result actions - * -1 final state with only ignorable actions - * - * Bits 5..3: - * The lowest byte value with non-ignorable actions is - * value<<5 (rounded down). - * - * Bits 2..0: - * The highest byte value with non-ignorable actions is - * (value<<5)&0x1f (rounded up). - */ - int8_t stateProps[MBCS_MAX_STATE_COUNT]; - int32_t state; - - uprv_memset(stateProps, -1, sizeof(stateProps)); - - /* recurse from state 0 and set all stateProps */ - getStateProp(mbcsTable->stateTable, stateProps, 0); - - for(state=0; state<mbcsTable->countStates; ++state) { - /*if(stateProps[state]==-1) { - printf("unused/unreachable <icu:state> %d\n", state); - }*/ - if(stateProps[state]>=0x40) { - /* start from each direct state */ - enumToU( - mbcsTable, stateProps, state, 0, 0, - callback, context, - pErrorCode); - } - } -} - -U_CFUNC void -ucnv_MBCSGetFilteredUnicodeSetForUnicode(const UConverterSharedData *sharedData, - const USetAdder *sa, - UConverterUnicodeSet which, - UConverterSetFilter filter, - UErrorCode *pErrorCode) { - const UConverterMBCSTable *mbcsTable; - const uint16_t *table; - - uint32_t st3; - uint16_t st1, maxStage1, st2; - - UChar32 c; - - /* enumerate the from-Unicode trie table */ - mbcsTable=&sharedData->mbcs; - table=mbcsTable->fromUnicodeTable; - if(mbcsTable->unicodeMask&UCNV_HAS_SUPPLEMENTARY) { - maxStage1=0x440; - } else { - maxStage1=0x40; - } - - c=0; /* keep track of the current code point while enumerating */ - - if(mbcsTable->outputType==MBCS_OUTPUT_1) { - const uint16_t *stage2, *stage3, *results; - uint16_t minValue; - - results=(const uint16_t *)mbcsTable->fromUnicodeBytes; - - /* - * Set a threshold variable for selecting which mappings to use. - * See ucnv_MBCSSingleFromBMPWithOffsets() and - * MBCS_SINGLE_RESULT_FROM_U() for details. - */ - if(which==UCNV_ROUNDTRIP_SET) { - /* use only roundtrips */ - minValue=0xf00; - } else /* UCNV_ROUNDTRIP_AND_FALLBACK_SET */ { - /* use all roundtrip and fallback results */ - minValue=0x800; - } - - for(st1=0; st1<maxStage1; ++st1) { - st2=table[st1]; - if(st2>maxStage1) { - stage2=table+st2; - for(st2=0; st2<64; ++st2) { - if((st3=stage2[st2])!=0) { - /* read the stage 3 block */ - stage3=results+st3; - - do { - if(*stage3++>=minValue) { - sa->add(sa->set, c); - } - } while((++c&0xf)!=0); - } else { - c+=16; /* empty stage 3 block */ - } - } - } else { - c+=1024; /* empty stage 2 block */ - } - } - } else { - const uint32_t *stage2; - const uint8_t *stage3, *bytes; - uint32_t st3Multiplier; - uint32_t value; - UBool useFallback; - - bytes=mbcsTable->fromUnicodeBytes; - - useFallback=(UBool)(which==UCNV_ROUNDTRIP_AND_FALLBACK_SET); - - switch(mbcsTable->outputType) { - case MBCS_OUTPUT_3: - case MBCS_OUTPUT_4_EUC: - st3Multiplier=3; - break; - case MBCS_OUTPUT_4: - st3Multiplier=4; - break; - default: - st3Multiplier=2; - break; - } - - for(st1=0; st1<maxStage1; ++st1) { - st2=table[st1]; - if(st2>(maxStage1>>1)) { - stage2=(const uint32_t *)table+st2; - for(st2=0; st2<64; ++st2) { - if((st3=stage2[st2])!=0) { - /* read the stage 3 block */ - stage3=bytes+st3Multiplier*16*(uint32_t)(uint16_t)st3; - - /* get the roundtrip flags for the stage 3 block */ - st3>>=16; - - /* - * Add code points for which the roundtrip flag is set, - * or which map to non-zero bytes if we use fallbacks. - * See ucnv_MBCSFromUnicodeWithOffsets() for details. - */ - switch(filter) { - case UCNV_SET_FILTER_NONE: - do { - if(st3&1) { - sa->add(sa->set, c); - stage3+=st3Multiplier; - } else if(useFallback) { - uint8_t b=0; - switch(st3Multiplier) { - case 4: - b|=*stage3++; - U_FALLTHROUGH; - case 3: - b|=*stage3++; - U_FALLTHROUGH; - case 2: - b|=stage3[0]|stage3[1]; - stage3+=2; - U_FALLTHROUGH; - default: - break; - } - if(b!=0) { - sa->add(sa->set, c); - } - } - st3>>=1; - } while((++c&0xf)!=0); - break; - case UCNV_SET_FILTER_DBCS_ONLY: - /* Ignore single-byte results (<0x100). */ - do { - if(((st3&1)!=0 || useFallback) && *((const uint16_t *)stage3)>=0x100) { - sa->add(sa->set, c); - } - st3>>=1; - stage3+=2; /* +=st3Multiplier */ - } while((++c&0xf)!=0); - break; - case UCNV_SET_FILTER_2022_CN: - /* Only add code points that map to CNS 11643 planes 1 & 2 for non-EXT ISO-2022-CN. */ - do { - if(((st3&1)!=0 || useFallback) && ((value=*stage3)==0x81 || value==0x82)) { - sa->add(sa->set, c); - } - st3>>=1; - stage3+=3; /* +=st3Multiplier */ - } while((++c&0xf)!=0); - break; - case UCNV_SET_FILTER_SJIS: - /* Only add code points that map to Shift-JIS codes corresponding to JIS X 0208. */ - do { - if(((st3&1)!=0 || useFallback) && (value=*((const uint16_t *)stage3))>=0x8140 && value<=0xeffc) { - sa->add(sa->set, c); - } - st3>>=1; - stage3+=2; /* +=st3Multiplier */ - } while((++c&0xf)!=0); - break; - case UCNV_SET_FILTER_GR94DBCS: - /* Only add code points that map to ISO 2022 GR 94 DBCS codes (each byte A1..FE). */ - do { - if( ((st3&1)!=0 || useFallback) && - (uint16_t)((value=*((const uint16_t *)stage3)) - 0xa1a1)<=(0xfefe - 0xa1a1) && - (uint8_t)(value-0xa1)<=(0xfe - 0xa1) - ) { - sa->add(sa->set, c); - } - st3>>=1; - stage3+=2; /* +=st3Multiplier */ - } while((++c&0xf)!=0); - break; - case UCNV_SET_FILTER_HZ: - /* Only add code points that are suitable for HZ DBCS (lead byte A1..FD). */ - do { - if( ((st3&1)!=0 || useFallback) && - (uint16_t)((value=*((const uint16_t *)stage3))-0xa1a1)<=(0xfdfe - 0xa1a1) && - (uint8_t)(value-0xa1)<=(0xfe - 0xa1) - ) { - sa->add(sa->set, c); - } - st3>>=1; - stage3+=2; /* +=st3Multiplier */ - } while((++c&0xf)!=0); - break; - default: - *pErrorCode=U_INTERNAL_PROGRAM_ERROR; - return; - } - } else { - c+=16; /* empty stage 3 block */ - } - } - } else { - c+=1024; /* empty stage 2 block */ - } - } - } - - ucnv_extGetUnicodeSet(sharedData, sa, which, filter, pErrorCode); -} - -U_CFUNC void -ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData, - const USetAdder *sa, - UConverterUnicodeSet which, - UErrorCode *pErrorCode) { - ucnv_MBCSGetFilteredUnicodeSetForUnicode( - sharedData, sa, which, - sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY ? - UCNV_SET_FILTER_DBCS_ONLY : - UCNV_SET_FILTER_NONE, - pErrorCode); -} - -static void U_CALLCONV -ucnv_MBCSGetUnicodeSet(const UConverter *cnv, - const USetAdder *sa, - UConverterUnicodeSet which, - UErrorCode *pErrorCode) { - if(cnv->options&_MBCS_OPTION_GB18030) { - sa->addRange(sa->set, 0, 0xd7ff); - sa->addRange(sa->set, 0xe000, 0x10ffff); - } else { - ucnv_MBCSGetUnicodeSetForUnicode(cnv->sharedData, sa, which, pErrorCode); - } -} - -/* conversion extensions for input not in the main table -------------------- */ - -/* - * Hardcoded extension handling for GB 18030. - * Definition of LINEAR macros and gb18030Ranges see near the beginning of the file. - * - * In the future, conversion extensions may handle m:n mappings and delta tables, - * see http://source.icu-project.org/repos/icu/icuhtml/trunk/design/conversion/conversion_extensions.html - * - * If an input character cannot be mapped, then these functions set an error - * code. The framework will then call the callback function. - */ - -/* - * @return if(U_FAILURE) return the code point for cnv->fromUChar32 - * else return 0 after output has been written to the target - */ -static UChar32 -_extFromU(UConverter *cnv, const UConverterSharedData *sharedData, - UChar32 cp, - const UChar **source, const UChar *sourceLimit, - uint8_t **target, const uint8_t *targetLimit, - int32_t **offsets, int32_t sourceIndex, - UBool flush, - UErrorCode *pErrorCode) { - const int32_t *cx; - - cnv->useSubChar1=FALSE; - - if( (cx=sharedData->mbcs.extIndexes)!=NULL && - ucnv_extInitialMatchFromU( - cnv, cx, - cp, source, sourceLimit, - (char **)target, (char *)targetLimit, - offsets, sourceIndex, - flush, - pErrorCode) - ) { - return 0; /* an extension mapping handled the input */ - } - - /* GB 18030 */ - if((cnv->options&_MBCS_OPTION_GB18030)!=0) { - const uint32_t *range; - int32_t i; - - range=gb18030Ranges[0]; - for(i=0; i<UPRV_LENGTHOF(gb18030Ranges); range+=4, ++i) { - if(range[0]<=(uint32_t)cp && (uint32_t)cp<=range[1]) { - /* found the Unicode code point, output the four-byte sequence for it */ - uint32_t linear; - char bytes[4]; - - /* get the linear value of the first GB 18030 code in this range */ - linear=range[2]-LINEAR_18030_BASE; - - /* add the offset from the beginning of the range */ - linear+=((uint32_t)cp-range[0]); - - /* turn this into a four-byte sequence */ - bytes[3]=(char)(0x30+linear%10); linear/=10; - bytes[2]=(char)(0x81+linear%126); linear/=126; - bytes[1]=(char)(0x30+linear%10); linear/=10; - bytes[0]=(char)(0x81+linear); - - /* output this sequence */ - ucnv_fromUWriteBytes(cnv, - bytes, 4, (char **)target, (char *)targetLimit, - offsets, sourceIndex, pErrorCode); - return 0; - } - } - } - - /* no mapping */ - *pErrorCode=U_INVALID_CHAR_FOUND; - return cp; -} - -/* - * Input sequence: cnv->toUBytes[0..length[ - * @return if(U_FAILURE) return the length (toULength, byteIndex) for the input - * else return 0 after output has been written to the target - */ -static int8_t -_extToU(UConverter *cnv, const UConverterSharedData *sharedData, - int8_t length, - const uint8_t **source, const uint8_t *sourceLimit, - UChar **target, const UChar *targetLimit, - int32_t **offsets, int32_t sourceIndex, - UBool flush, - UErrorCode *pErrorCode) { - const int32_t *cx; - - if( (cx=sharedData->mbcs.extIndexes)!=NULL && - ucnv_extInitialMatchToU( - cnv, cx, - length, (const char **)source, (const char *)sourceLimit, - target, targetLimit, - offsets, sourceIndex, - flush, - pErrorCode) - ) { - return 0; /* an extension mapping handled the input */ - } - - /* GB 18030 */ - if(length==4 && (cnv->options&_MBCS_OPTION_GB18030)!=0) { - const uint32_t *range; - uint32_t linear; - int32_t i; - - linear=LINEAR_18030(cnv->toUBytes[0], cnv->toUBytes[1], cnv->toUBytes[2], cnv->toUBytes[3]); - range=gb18030Ranges[0]; - for(i=0; i<UPRV_LENGTHOF(gb18030Ranges); range+=4, ++i) { - if(range[2]<=linear && linear<=range[3]) { - /* found the sequence, output the Unicode code point for it */ - *pErrorCode=U_ZERO_ERROR; - - /* add the linear difference between the input and start sequences to the start code point */ - linear=range[0]+(linear-range[2]); - - /* output this code point */ - ucnv_toUWriteCodePoint(cnv, linear, target, targetLimit, offsets, sourceIndex, pErrorCode); - - return 0; - } - } - } - - /* no mapping */ - *pErrorCode=U_INVALID_CHAR_FOUND; - return length; -} - -/* EBCDIC swap LF<->NL ------------------------------------------------------ */ - -/* - * This code modifies a standard EBCDIC<->Unicode mapping table for - * OS/390 (z/OS) Unix System Services (Open Edition). - * The difference is in the mapping of Line Feed and New Line control codes: - * Standard EBCDIC maps - * - * <U000A> \x25 |0 - * <U0085> \x15 |0 - * - * but OS/390 USS EBCDIC swaps the control codes for LF and NL, - * mapping - * - * <U000A> \x15 |0 - * <U0085> \x25 |0 - * - * This code modifies a loaded standard EBCDIC<->Unicode mapping table - * by copying it into allocated memory and swapping the LF and NL values. - * It allows to support the same EBCDIC charset in both versions without - * duplicating the entire installed table. - */ - -/* standard EBCDIC codes */ -#define EBCDIC_LF 0x25 -#define EBCDIC_NL 0x15 - -/* standard EBCDIC codes with roundtrip flag as stored in Unicode-to-single-byte tables */ -#define EBCDIC_RT_LF 0xf25 -#define EBCDIC_RT_NL 0xf15 - -/* Unicode code points */ -#define U_LF 0x0a -#define U_NL 0x85 - -static UBool -_EBCDICSwapLFNL(UConverterSharedData *sharedData, UErrorCode *pErrorCode) { - UConverterMBCSTable *mbcsTable; - - const uint16_t *table, *results; - const uint8_t *bytes; - - int32_t (*newStateTable)[256]; - uint16_t *newResults; - uint8_t *p; - char *name; - - uint32_t stage2Entry; - uint32_t size, sizeofFromUBytes; - - mbcsTable=&sharedData->mbcs; - - table=mbcsTable->fromUnicodeTable; - bytes=mbcsTable->fromUnicodeBytes; - results=(const uint16_t *)bytes; - - /* - * Check that this is an EBCDIC table with SBCS portion - - * SBCS or EBCDIC_STATEFUL with standard EBCDIC LF and NL mappings. - * - * If not, ignore the option. Options are always ignored if they do not apply. - */ - if(!( - (mbcsTable->outputType==MBCS_OUTPUT_1 || mbcsTable->outputType==MBCS_OUTPUT_2_SISO) && - mbcsTable->stateTable[0][EBCDIC_LF]==MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_LF) && - mbcsTable->stateTable[0][EBCDIC_NL]==MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_NL) - )) { - return FALSE; - } - - if(mbcsTable->outputType==MBCS_OUTPUT_1) { - if(!( - EBCDIC_RT_LF==MBCS_SINGLE_RESULT_FROM_U(table, results, U_LF) && - EBCDIC_RT_NL==MBCS_SINGLE_RESULT_FROM_U(table, results, U_NL) - )) { - return FALSE; - } - } else /* MBCS_OUTPUT_2_SISO */ { - stage2Entry=MBCS_STAGE_2_FROM_U(table, U_LF); - if(!( - MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, U_LF)!=0 && - EBCDIC_LF==MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, U_LF) - )) { - return FALSE; - } - - stage2Entry=MBCS_STAGE_2_FROM_U(table, U_NL); - if(!( - MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, U_NL)!=0 && - EBCDIC_NL==MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, U_NL) - )) { - return FALSE; - } - } - - if(mbcsTable->fromUBytesLength>0) { - /* - * We _know_ the number of bytes in the fromUnicodeBytes array - * starting with header.version 4.1. - */ - sizeofFromUBytes=mbcsTable->fromUBytesLength; - } else { - /* - * Otherwise: - * There used to be code to enumerate the fromUnicode - * trie and find the highest entry, but it was removed in ICU 3.2 - * because it was not tested and caused a low code coverage number. - * See Jitterbug 3674. - * This affects only some .cnv file formats with a header.version - * below 4.1, and only when swaplfnl is requested. - * - * ucnvmbcs.c revision 1.99 is the last one with the - * ucnv_MBCSSizeofFromUBytes() function. - */ - *pErrorCode=U_INVALID_FORMAT_ERROR; - return FALSE; - } - - /* - * The table has an appropriate format. - * Allocate and build - * - a modified to-Unicode state table - * - a modified from-Unicode output array - * - a converter name string with the swap option appended - */ - size= - mbcsTable->countStates*1024+ - sizeofFromUBytes+ - UCNV_MAX_CONVERTER_NAME_LENGTH+20; - p=(uint8_t *)uprv_malloc(size); - if(p==NULL) { - *pErrorCode=U_MEMORY_ALLOCATION_ERROR; - return FALSE; - } - - /* copy and modify the to-Unicode state table */ - newStateTable=(int32_t (*)[256])p; - uprv_memcpy(newStateTable, mbcsTable->stateTable, mbcsTable->countStates*1024); - - newStateTable[0][EBCDIC_LF]=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_NL); - newStateTable[0][EBCDIC_NL]=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_LF); - - /* copy and modify the from-Unicode result table */ - newResults=(uint16_t *)newStateTable[mbcsTable->countStates]; - uprv_memcpy(newResults, bytes, sizeofFromUBytes); - - /* conveniently, the table access macros work on the left side of expressions */ - if(mbcsTable->outputType==MBCS_OUTPUT_1) { - MBCS_SINGLE_RESULT_FROM_U(table, newResults, U_LF)=EBCDIC_RT_NL; - MBCS_SINGLE_RESULT_FROM_U(table, newResults, U_NL)=EBCDIC_RT_LF; - } else /* MBCS_OUTPUT_2_SISO */ { - stage2Entry=MBCS_STAGE_2_FROM_U(table, U_LF); - MBCS_VALUE_2_FROM_STAGE_2(newResults, stage2Entry, U_LF)=EBCDIC_NL; - - stage2Entry=MBCS_STAGE_2_FROM_U(table, U_NL); - MBCS_VALUE_2_FROM_STAGE_2(newResults, stage2Entry, U_NL)=EBCDIC_LF; - } - - /* set the canonical converter name */ - name=(char *)newResults+sizeofFromUBytes; - uprv_strcpy(name, sharedData->staticData->name); - uprv_strcat(name, UCNV_SWAP_LFNL_OPTION_STRING); - - /* set the pointers */ - umtx_lock(NULL); - if(mbcsTable->swapLFNLStateTable==NULL) { - mbcsTable->swapLFNLStateTable=newStateTable; - mbcsTable->swapLFNLFromUnicodeBytes=(uint8_t *)newResults; - mbcsTable->swapLFNLName=name; - - newStateTable=NULL; - } - umtx_unlock(NULL); - - /* release the allocated memory if another thread beat us to it */ - if(newStateTable!=NULL) { - uprv_free(newStateTable); - } - return TRUE; -} - -/* reconstitute omitted fromUnicode data ------------------------------------ */ - -/* for details, compare with genmbcs.c MBCSAddFromUnicode() and transformEUC() */ -static UBool U_CALLCONV -writeStage3Roundtrip(const void *context, uint32_t value, UChar32 codePoints[32]) { - UConverterMBCSTable *mbcsTable=(UConverterMBCSTable *)context; - const uint16_t *table; - uint32_t *stage2; - uint8_t *bytes, *p; - UChar32 c; - int32_t i, st3; - - table=mbcsTable->fromUnicodeTable; - bytes=(uint8_t *)mbcsTable->fromUnicodeBytes; - - /* for EUC outputTypes, modify the value like genmbcs.c's transformEUC() */ - switch(mbcsTable->outputType) { - case MBCS_OUTPUT_3_EUC: - if(value<=0xffff) { - /* short sequences are stored directly */ - /* code set 0 or 1 */ - } else if(value<=0x8effff) { - /* code set 2 */ - value&=0x7fff; - } else /* first byte is 0x8f */ { - /* code set 3 */ - value&=0xff7f; - } - break; - case MBCS_OUTPUT_4_EUC: - if(value<=0xffffff) { - /* short sequences are stored directly */ - /* code set 0 or 1 */ - } else if(value<=0x8effffff) { - /* code set 2 */ - value&=0x7fffff; - } else /* first byte is 0x8f */ { - /* code set 3 */ - value&=0xff7fff; - } - break; - default: - break; - } - - for(i=0; i<=0x1f; ++value, ++i) { - c=codePoints[i]; - if(c<0) { - continue; - } - - /* locate the stage 2 & 3 data */ - stage2=((uint32_t *)table)+table[c>>10]+((c>>4)&0x3f); - p=bytes; - st3=(int32_t)(uint16_t)*stage2*16+(c&0xf); - - /* write the codepage bytes into stage 3 */ - switch(mbcsTable->outputType) { - case MBCS_OUTPUT_3: - case MBCS_OUTPUT_4_EUC: - p+=st3*3; - p[0]=(uint8_t)(value>>16); - p[1]=(uint8_t)(value>>8); - p[2]=(uint8_t)value; - break; - case MBCS_OUTPUT_4: - ((uint32_t *)p)[st3]=value; - break; - default: - /* 2 bytes per character */ - ((uint16_t *)p)[st3]=(uint16_t)value; - break; - } - - /* set the roundtrip flag */ - *stage2|=(1UL<<(16+(c&0xf))); - } - return TRUE; - } - -static void -reconstituteData(UConverterMBCSTable *mbcsTable, - uint32_t stage1Length, uint32_t stage2Length, - uint32_t fullStage2Length, /* lengths are numbers of units, not bytes */ - UErrorCode *pErrorCode) { - uint16_t *stage1; - uint32_t *stage2; - uint32_t dataLength=stage1Length*2+fullStage2Length*4+mbcsTable->fromUBytesLength; - mbcsTable->reconstitutedData=(uint8_t *)uprv_malloc(dataLength); - if(mbcsTable->reconstitutedData==NULL) { - *pErrorCode=U_MEMORY_ALLOCATION_ERROR; - return; - } - uprv_memset(mbcsTable->reconstitutedData, 0, dataLength); - - /* copy existing data and reroute the pointers */ - stage1=(uint16_t *)mbcsTable->reconstitutedData; - uprv_memcpy(stage1, mbcsTable->fromUnicodeTable, stage1Length*2); - - stage2=(uint32_t *)(stage1+stage1Length); - uprv_memcpy(stage2+(fullStage2Length-stage2Length), - mbcsTable->fromUnicodeTable+stage1Length, - stage2Length*4); - - mbcsTable->fromUnicodeTable=stage1; - mbcsTable->fromUnicodeBytes=(uint8_t *)(stage2+fullStage2Length); - - /* indexes into stage 2 count from the bottom of the fromUnicodeTable */ - stage2=(uint32_t *)stage1; - - /* reconstitute the initial part of stage 2 from the mbcsIndex */ - { - int32_t stageUTF8Length=((int32_t)mbcsTable->maxFastUChar+1)>>6; - int32_t stageUTF8Index=0; - int32_t st1, st2, st3, i; - - for(st1=0; stageUTF8Index<stageUTF8Length; ++st1) { - st2=stage1[st1]; - if(st2!=(int32_t)stage1Length/2) { - /* each stage 2 block has 64 entries corresponding to 16 entries in the mbcsIndex */ - for(i=0; i<16; ++i) { - st3=mbcsTable->mbcsIndex[stageUTF8Index++]; - if(st3!=0) { - /* an stage 2 entry's index is per stage 3 16-block, not per stage 3 entry */ - st3>>=4; - /* - * 4 stage 2 entries point to 4 consecutive stage 3 16-blocks which are - * allocated together as a single 64-block for access from the mbcsIndex - */ - stage2[st2++]=st3++; - stage2[st2++]=st3++; - stage2[st2++]=st3++; - stage2[st2++]=st3; - } else { - /* no stage 3 block, skip */ - st2+=4; - } - } - } else { - /* no stage 2 block, skip */ - stageUTF8Index+=16; - } - } - } - - /* reconstitute fromUnicodeBytes with roundtrips from toUnicode data */ - ucnv_MBCSEnumToUnicode(mbcsTable, writeStage3Roundtrip, mbcsTable, pErrorCode); -} - -/* MBCS setup functions ----------------------------------------------------- */ - -static void U_CALLCONV -ucnv_MBCSLoad(UConverterSharedData *sharedData, - UConverterLoadArgs *pArgs, - const uint8_t *raw, - UErrorCode *pErrorCode) { - UDataInfo info; - UConverterMBCSTable *mbcsTable=&sharedData->mbcs; - _MBCSHeader *header=(_MBCSHeader *)raw; - uint32_t offset; - uint32_t headerLength; - UBool noFromU=FALSE; - - if(header->version[0]==4) { - headerLength=MBCS_HEADER_V4_LENGTH; - } else if(header->version[0]==5 && header->version[1]>=3 && - (header->options&MBCS_OPT_UNKNOWN_INCOMPATIBLE_MASK)==0) { - headerLength=header->options&MBCS_OPT_LENGTH_MASK; - noFromU=(UBool)((header->options&MBCS_OPT_NO_FROM_U)!=0); - } else { - *pErrorCode=U_INVALID_TABLE_FORMAT; - return; - } - - mbcsTable->outputType=(uint8_t)header->flags; - if(noFromU && mbcsTable->outputType==MBCS_OUTPUT_1) { - *pErrorCode=U_INVALID_TABLE_FORMAT; - return; - } - - /* extension data, header version 4.2 and higher */ - offset=header->flags>>8; - if(offset!=0) { - mbcsTable->extIndexes=(const int32_t *)(raw+offset); - } - - if(mbcsTable->outputType==MBCS_OUTPUT_EXT_ONLY) { - UConverterLoadArgs args=UCNV_LOAD_ARGS_INITIALIZER; - UConverterSharedData *baseSharedData; - const int32_t *extIndexes; - const char *baseName; - - /* extension-only file, load the base table and set values appropriately */ - if((extIndexes=mbcsTable->extIndexes)==NULL) { - /* extension-only file without extension */ - *pErrorCode=U_INVALID_TABLE_FORMAT; - return; - } - - if(pArgs->nestedLoads!=1) { - /* an extension table must not be loaded as a base table */ - *pErrorCode=U_INVALID_TABLE_FILE; - return; - } - - /* load the base table */ - baseName=(const char *)header+headerLength*4; - if(0==uprv_strcmp(baseName, sharedData->staticData->name)) { - /* forbid loading this same extension-only file */ - *pErrorCode=U_INVALID_TABLE_FORMAT; - return; - } - - /* TODO parse package name out of the prefix of the base name in the extension .cnv file? */ - args.size=sizeof(UConverterLoadArgs); - args.nestedLoads=2; - args.onlyTestIsLoadable=pArgs->onlyTestIsLoadable; - args.reserved=pArgs->reserved; - args.options=pArgs->options; - args.pkg=pArgs->pkg; - args.name=baseName; - baseSharedData=ucnv_load(&args, pErrorCode); - if(U_FAILURE(*pErrorCode)) { - return; - } - if( baseSharedData->staticData->conversionType!=UCNV_MBCS || - baseSharedData->mbcs.baseSharedData!=NULL - ) { - ucnv_unload(baseSharedData); - *pErrorCode=U_INVALID_TABLE_FORMAT; - return; - } - if(pArgs->onlyTestIsLoadable) { - /* - * Exit as soon as we know that we can load the converter - * and the format is valid and supported. - * The worst that can happen in the following code is a memory - * allocation error. - */ - ucnv_unload(baseSharedData); - return; - } - - /* copy the base table data */ - uprv_memcpy(mbcsTable, &baseSharedData->mbcs, sizeof(UConverterMBCSTable)); - - /* overwrite values with relevant ones for the extension converter */ - mbcsTable->baseSharedData=baseSharedData; - mbcsTable->extIndexes=extIndexes; - - /* - * It would be possible to share the swapLFNL data with a base converter, - * but the generated name would have to be different, and the memory - * would have to be free'd only once. - * It is easier to just create the data for the extension converter - * separately when it is requested. - */ - mbcsTable->swapLFNLStateTable=NULL; - mbcsTable->swapLFNLFromUnicodeBytes=NULL; - mbcsTable->swapLFNLName=NULL; - - /* - * The reconstitutedData must be deleted only when the base converter - * is unloaded. - */ - mbcsTable->reconstitutedData=NULL; - - /* - * Set a special, runtime-only outputType if the extension converter - * is a DBCS version of a base converter that also maps single bytes. - */ - if( sharedData->staticData->conversionType==UCNV_DBCS || - (sharedData->staticData->conversionType==UCNV_MBCS && - sharedData->staticData->minBytesPerChar>=2) - ) { - if(baseSharedData->mbcs.outputType==MBCS_OUTPUT_2_SISO) { - /* the base converter is SI/SO-stateful */ - int32_t entry; - - /* get the dbcs state from the state table entry for SO=0x0e */ - entry=mbcsTable->stateTable[0][0xe]; - if( MBCS_ENTRY_IS_FINAL(entry) && - MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_CHANGE_ONLY && - MBCS_ENTRY_FINAL_STATE(entry)!=0 - ) { - mbcsTable->dbcsOnlyState=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); - - mbcsTable->outputType=MBCS_OUTPUT_DBCS_ONLY; - } - } else if( - baseSharedData->staticData->conversionType==UCNV_MBCS && - baseSharedData->staticData->minBytesPerChar==1 && - baseSharedData->staticData->maxBytesPerChar==2 && - mbcsTable->countStates<=127 - ) { - /* non-stateful base converter, need to modify the state table */ - int32_t (*newStateTable)[256]; - int32_t *state; - int32_t i, count; - - /* allocate a new state table and copy the base state table contents */ - count=mbcsTable->countStates; - newStateTable=(int32_t (*)[256])uprv_malloc((count+1)*1024); - if(newStateTable==NULL) { - ucnv_unload(baseSharedData); - *pErrorCode=U_MEMORY_ALLOCATION_ERROR; - return; - } - - uprv_memcpy(newStateTable, mbcsTable->stateTable, count*1024); - - /* change all final single-byte entries to go to a new all-illegal state */ - state=newStateTable[0]; - for(i=0; i<256; ++i) { - if(MBCS_ENTRY_IS_FINAL(state[i])) { - state[i]=MBCS_ENTRY_TRANSITION(count, 0); - } - } - - /* build the new all-illegal state */ - state=newStateTable[count]; - for(i=0; i<256; ++i) { - state[i]=MBCS_ENTRY_FINAL(0, MBCS_STATE_ILLEGAL, 0); - } - mbcsTable->stateTable=(const int32_t (*)[256])newStateTable; - mbcsTable->countStates=(uint8_t)(count+1); - mbcsTable->stateTableOwned=TRUE; - - mbcsTable->outputType=MBCS_OUTPUT_DBCS_ONLY; - } - } - - /* - * unlike below for files with base tables, do not get the unicodeMask - * from the sharedData; instead, use the base table's unicodeMask, - * which we copied in the memcpy above; - * this is necessary because the static data unicodeMask, especially - * the UCNV_HAS_SUPPLEMENTARY flag, is part of the base table data - */ - } else { - /* conversion file with a base table; an additional extension table is optional */ - /* make sure that the output type is known */ - switch(mbcsTable->outputType) { - case MBCS_OUTPUT_1: - case MBCS_OUTPUT_2: - case MBCS_OUTPUT_3: - case MBCS_OUTPUT_4: - case MBCS_OUTPUT_3_EUC: - case MBCS_OUTPUT_4_EUC: - case MBCS_OUTPUT_2_SISO: - /* OK */ - break; - default: - *pErrorCode=U_INVALID_TABLE_FORMAT; - return; - } - if(pArgs->onlyTestIsLoadable) { - /* - * Exit as soon as we know that we can load the converter - * and the format is valid and supported. - * The worst that can happen in the following code is a memory - * allocation error. - */ - return; - } - - mbcsTable->countStates=(uint8_t)header->countStates; - mbcsTable->countToUFallbacks=header->countToUFallbacks; - mbcsTable->stateTable=(const int32_t (*)[256])(raw+headerLength*4); - mbcsTable->toUFallbacks=(const _MBCSToUFallback *)(mbcsTable->stateTable+header->countStates); - mbcsTable->unicodeCodeUnits=(const uint16_t *)(raw+header->offsetToUCodeUnits); - - mbcsTable->fromUnicodeTable=(const uint16_t *)(raw+header->offsetFromUTable); - mbcsTable->fromUnicodeBytes=(const uint8_t *)(raw+header->offsetFromUBytes); - mbcsTable->fromUBytesLength=header->fromUBytesLength; - - /* - * converter versions 6.1 and up contain a unicodeMask that is - * used here to select the most efficient function implementations - */ - info.size=sizeof(UDataInfo); - udata_getInfo((UDataMemory *)sharedData->dataMemory, &info); - if(info.formatVersion[0]>6 || (info.formatVersion[0]==6 && info.formatVersion[1]>=1)) { - /* mask off possible future extensions to be safe */ - mbcsTable->unicodeMask=(uint8_t)(sharedData->staticData->unicodeMask&3); - } else { - /* for older versions, assume worst case: contains anything possible (prevent over-optimizations) */ - mbcsTable->unicodeMask=UCNV_HAS_SUPPLEMENTARY|UCNV_HAS_SURROGATES; - } - - /* - * _MBCSHeader.version 4.3 adds utf8Friendly data structures. - * Check for the header version, SBCS vs. MBCS, and for whether the - * data structures are optimized for code points as high as what the - * runtime code is designed for. - * The implementation does not handle mapping tables with entries for - * unpaired surrogates. - */ - if( header->version[1]>=3 && - (mbcsTable->unicodeMask&UCNV_HAS_SURROGATES)==0 && - (mbcsTable->countStates==1 ? - (header->version[2]>=(SBCS_FAST_MAX>>8)) : - (header->version[2]>=(MBCS_FAST_MAX>>8)) - ) - ) { - mbcsTable->utf8Friendly=TRUE; - - if(mbcsTable->countStates==1) { - /* - * SBCS: Stage 3 is allocated in 64-entry blocks for U+0000..SBCS_FAST_MAX or higher. - * Build a table with indexes to each block, to be used instead of - * the regular stage 1/2 table. - */ - int32_t i; - for(i=0; i<(SBCS_FAST_LIMIT>>6); ++i) { - mbcsTable->sbcsIndex[i]=mbcsTable->fromUnicodeTable[mbcsTable->fromUnicodeTable[i>>4]+((i<<2)&0x3c)]; - } - /* set SBCS_FAST_MAX to reflect the reach of sbcsIndex[] even if header->version[2]>(SBCS_FAST_MAX>>8) */ - mbcsTable->maxFastUChar=SBCS_FAST_MAX; - } else { - /* - * MBCS: Stage 3 is allocated in 64-entry blocks for U+0000..MBCS_FAST_MAX or higher. - * The .cnv file is prebuilt with an additional stage table with indexes - * to each block. - */ - mbcsTable->mbcsIndex=(const uint16_t *) - (mbcsTable->fromUnicodeBytes+ - (noFromU ? 0 : mbcsTable->fromUBytesLength)); - mbcsTable->maxFastUChar=(((UChar)header->version[2])<<8)|0xff; - } - } - - /* calculate a bit set of 4 ASCII characters per bit that round-trip to ASCII bytes */ - { - uint32_t asciiRoundtrips=0xffffffff; - int32_t i; - - for(i=0; i<0x80; ++i) { - if(mbcsTable->stateTable[0][i]!=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, i)) { - asciiRoundtrips&=~((uint32_t)1<<(i>>2)); - } - } - mbcsTable->asciiRoundtrips=asciiRoundtrips; - } - - if(noFromU) { - uint32_t stage1Length= - mbcsTable->unicodeMask&UCNV_HAS_SUPPLEMENTARY ? - 0x440 : 0x40; - uint32_t stage2Length= - (header->offsetFromUBytes-header->offsetFromUTable)/4- - stage1Length/2; - reconstituteData(mbcsTable, stage1Length, stage2Length, header->fullStage2Length, pErrorCode); - } - } - - /* Set the impl pointer here so that it is set for both extension-only and base tables. */ - if(mbcsTable->utf8Friendly) { - if(mbcsTable->countStates==1) { - sharedData->impl=&_SBCSUTF8Impl; - } else { - if(mbcsTable->outputType==MBCS_OUTPUT_2) { - sharedData->impl=&_DBCSUTF8Impl; - } - } - } - - if(mbcsTable->outputType==MBCS_OUTPUT_DBCS_ONLY || mbcsTable->outputType==MBCS_OUTPUT_2_SISO) { - /* - * MBCS_OUTPUT_DBCS_ONLY: No SBCS mappings, therefore ASCII does not roundtrip. - * MBCS_OUTPUT_2_SISO: Bypass the ASCII fastpath to handle prevLength correctly. - */ - mbcsTable->asciiRoundtrips=0; - } -} - -static void U_CALLCONV -ucnv_MBCSUnload(UConverterSharedData *sharedData) { - UConverterMBCSTable *mbcsTable=&sharedData->mbcs; - - if(mbcsTable->swapLFNLStateTable!=NULL) { - uprv_free(mbcsTable->swapLFNLStateTable); - } - if(mbcsTable->stateTableOwned) { - uprv_free((void *)mbcsTable->stateTable); - } - if(mbcsTable->baseSharedData!=NULL) { - ucnv_unload(mbcsTable->baseSharedData); - } - if(mbcsTable->reconstitutedData!=NULL) { - uprv_free(mbcsTable->reconstitutedData); - } -} - -static void U_CALLCONV -ucnv_MBCSOpen(UConverter *cnv, - UConverterLoadArgs *pArgs, - UErrorCode *pErrorCode) { - UConverterMBCSTable *mbcsTable; - const int32_t *extIndexes; - uint8_t outputType; - int8_t maxBytesPerUChar; - - if(pArgs->onlyTestIsLoadable) { - return; - } - - mbcsTable=&cnv->sharedData->mbcs; - outputType=mbcsTable->outputType; - - if(outputType==MBCS_OUTPUT_DBCS_ONLY) { - /* the swaplfnl option does not apply, remove it */ - cnv->options=pArgs->options&=~UCNV_OPTION_SWAP_LFNL; - } - - if((pArgs->options&UCNV_OPTION_SWAP_LFNL)!=0) { - /* do this because double-checked locking is broken */ - UBool isCached; - - umtx_lock(NULL); - isCached=mbcsTable->swapLFNLStateTable!=NULL; - umtx_unlock(NULL); - - if(!isCached) { - if(!_EBCDICSwapLFNL(cnv->sharedData, pErrorCode)) { - if(U_FAILURE(*pErrorCode)) { - return; /* something went wrong */ - } - - /* the option does not apply, remove it */ - cnv->options=pArgs->options&=~UCNV_OPTION_SWAP_LFNL; - } - } - } - - if(uprv_strstr(pArgs->name, "18030")!=NULL) { - if(uprv_strstr(pArgs->name, "gb18030")!=NULL || uprv_strstr(pArgs->name, "GB18030")!=NULL) { - /* set a flag for GB 18030 mode, which changes the callback behavior */ - cnv->options|=_MBCS_OPTION_GB18030; - } - } else if((uprv_strstr(pArgs->name, "KEIS")!=NULL) || (uprv_strstr(pArgs->name, "keis")!=NULL)) { - /* set a flag for KEIS converter, which changes the SI/SO character sequence */ - cnv->options|=_MBCS_OPTION_KEIS; - } else if((uprv_strstr(pArgs->name, "JEF")!=NULL) || (uprv_strstr(pArgs->name, "jef")!=NULL)) { - /* set a flag for JEF converter, which changes the SI/SO character sequence */ - cnv->options|=_MBCS_OPTION_JEF; - } else if((uprv_strstr(pArgs->name, "JIPS")!=NULL) || (uprv_strstr(pArgs->name, "jips")!=NULL)) { - /* set a flag for JIPS converter, which changes the SI/SO character sequence */ - cnv->options|=_MBCS_OPTION_JIPS; - } - - /* fix maxBytesPerUChar depending on outputType and options etc. */ - if(outputType==MBCS_OUTPUT_2_SISO) { - cnv->maxBytesPerUChar=3; /* SO+DBCS */ - } - - extIndexes=mbcsTable->extIndexes; - if(extIndexes!=NULL) { - maxBytesPerUChar=(int8_t)UCNV_GET_MAX_BYTES_PER_UCHAR(extIndexes); - if(outputType==MBCS_OUTPUT_2_SISO) { - ++maxBytesPerUChar; /* SO + multiple DBCS */ - } - - if(maxBytesPerUChar>cnv->maxBytesPerUChar) { - cnv->maxBytesPerUChar=maxBytesPerUChar; - } - } - -#if 0 - /* - * documentation of UConverter fields used for status - * all of these fields are (re)set to 0 by ucnv_bld.c and ucnv_reset() - */ - - /* toUnicode */ - cnv->toUnicodeStatus=0; /* offset */ - cnv->mode=0; /* state */ - cnv->toULength=0; /* byteIndex */ - - /* fromUnicode */ - cnv->fromUChar32=0; - cnv->fromUnicodeStatus=1; /* prevLength */ -#endif -} - -U_CDECL_BEGIN - -static const char* U_CALLCONV -ucnv_MBCSGetName(const UConverter *cnv) { - if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0 && cnv->sharedData->mbcs.swapLFNLName!=NULL) { - return cnv->sharedData->mbcs.swapLFNLName; - } else { - return cnv->sharedData->staticData->name; - } -} -U_CDECL_END - - -/* MBCS-to-Unicode conversion functions ------------------------------------- */ - -static UChar32 U_CALLCONV -ucnv_MBCSGetFallback(UConverterMBCSTable *mbcsTable, uint32_t offset) { - const _MBCSToUFallback *toUFallbacks; - uint32_t i, start, limit; - - limit=mbcsTable->countToUFallbacks; - if(limit>0) { - /* do a binary search for the fallback mapping */ - toUFallbacks=mbcsTable->toUFallbacks; - start=0; - while(start<limit-1) { - i=(start+limit)/2; - if(offset<toUFallbacks[i].offset) { - limit=i; - } else { - start=i; - } - } - - /* did we really find it? */ - if(offset==toUFallbacks[start].offset) { - return toUFallbacks[start].codePoint; - } - } - - return 0xfffe; -} - -/* This version of ucnv_MBCSToUnicodeWithOffsets() is optimized for single-byte, single-state codepages. */ -static void -ucnv_MBCSSingleToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, - UErrorCode *pErrorCode) { - UConverter *cnv; - const uint8_t *source, *sourceLimit; - UChar *target; - const UChar *targetLimit; - int32_t *offsets; - - const int32_t (*stateTable)[256]; - - int32_t sourceIndex; - - int32_t entry; - UChar c; - uint8_t action; - - /* set up the local pointers */ - cnv=pArgs->converter; - source=(const uint8_t *)pArgs->source; - sourceLimit=(const uint8_t *)pArgs->sourceLimit; - target=pArgs->target; - targetLimit=pArgs->targetLimit; - offsets=pArgs->offsets; - - if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { - stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; - } else { - stateTable=cnv->sharedData->mbcs.stateTable; - } - - /* sourceIndex=-1 if the current character began in the previous buffer */ - sourceIndex=0; - - /* conversion loop */ - while(source<sourceLimit) { - /* - * This following test is to see if available input would overflow the output. - * It does not catch output of more than one code unit that - * overflows as a result of a surrogate pair or callback output - * from the last source byte. - * Therefore, those situations also test for overflows and will - * then break the loop, too. - */ - if(target>=targetLimit) { - /* target is full */ - *pErrorCode=U_BUFFER_OVERFLOW_ERROR; - break; - } - - entry=stateTable[0][*source++]; - /* MBCS_ENTRY_IS_FINAL(entry) */ - - /* test the most common case first */ - if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { - /* output BMP code point */ - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - - /* normal end of action codes: prepare for a new character */ - ++sourceIndex; - continue; - } - - /* - * An if-else-if chain provides more reliable performance for - * the most common cases compared to a switch. - */ - action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); - if(action==MBCS_STATE_VALID_DIRECT_20 || - (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv)) - ) { - entry=MBCS_ENTRY_FINAL_VALUE(entry); - /* output surrogate pair */ - *target++=(UChar)(0xd800|(UChar)(entry>>10)); - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - c=(UChar)(0xdc00|(UChar)(entry&0x3ff)); - if(target<targetLimit) { - *target++=c; - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - } else { - /* target overflow */ - cnv->UCharErrorBuffer[0]=c; - cnv->UCharErrorBufferLength=1; - *pErrorCode=U_BUFFER_OVERFLOW_ERROR; - break; - } - - ++sourceIndex; - continue; - } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { - if(UCNV_TO_U_USE_FALLBACK(cnv)) { - /* output BMP code point */ - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - - ++sourceIndex; - continue; - } - } else if(action==MBCS_STATE_UNASSIGNED) { - /* just fall through */ - } else if(action==MBCS_STATE_ILLEGAL) { - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - } else { - /* reserved, must never occur */ - ++sourceIndex; - continue; - } - - if(U_FAILURE(*pErrorCode)) { - /* callback(illegal) */ - break; - } else /* unassigned sequences indicated with byteIndex>0 */ { - /* try an extension mapping */ - pArgs->source=(const char *)source; - cnv->toUBytes[0]=*(source-1); - cnv->toULength=_extToU(cnv, cnv->sharedData, - 1, &source, sourceLimit, - &target, targetLimit, - &offsets, sourceIndex, - pArgs->flush, - pErrorCode); - sourceIndex+=1+(int32_t)(source-(const uint8_t *)pArgs->source); - - if(U_FAILURE(*pErrorCode)) { - /* not mappable or buffer overflow */ - break; - } - } - } - - /* write back the updated pointers */ - pArgs->source=(const char *)source; - pArgs->target=target; - pArgs->offsets=offsets; -} - -/* - * This version of ucnv_MBCSSingleToUnicodeWithOffsets() is optimized for single-byte, single-state codepages - * that only map to and from the BMP. - * In addition to single-byte optimizations, the offset calculations - * become much easier. - */ -static void -ucnv_MBCSSingleToBMPWithOffsets(UConverterToUnicodeArgs *pArgs, - UErrorCode *pErrorCode) { - UConverter *cnv; - const uint8_t *source, *sourceLimit, *lastSource; - UChar *target; - int32_t targetCapacity, length; - int32_t *offsets; - - const int32_t (*stateTable)[256]; - - int32_t sourceIndex; - - int32_t entry; - uint8_t action; - - /* set up the local pointers */ - cnv=pArgs->converter; - source=(const uint8_t *)pArgs->source; - sourceLimit=(const uint8_t *)pArgs->sourceLimit; - target=pArgs->target; - targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); - offsets=pArgs->offsets; - - if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { - stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; - } else { - stateTable=cnv->sharedData->mbcs.stateTable; - } - - /* sourceIndex=-1 if the current character began in the previous buffer */ - sourceIndex=0; - lastSource=source; - - /* - * since the conversion here is 1:1 UChar:uint8_t, we need only one counter - * for the minimum of the sourceLength and targetCapacity - */ - length=(int32_t)(sourceLimit-source); - if(length<targetCapacity) { - targetCapacity=length; - } - -#if MBCS_UNROLL_SINGLE_TO_BMP - /* unrolling makes it faster on Pentium III/Windows 2000 */ - /* unroll the loop with the most common case */ -unrolled: - if(targetCapacity>=16) { - int32_t count, loops, oredEntries; - - loops=count=targetCapacity>>4; - do { - oredEntries=entry=stateTable[0][*source++]; - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - oredEntries|=entry=stateTable[0][*source++]; - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - oredEntries|=entry=stateTable[0][*source++]; - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - oredEntries|=entry=stateTable[0][*source++]; - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - oredEntries|=entry=stateTable[0][*source++]; - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - oredEntries|=entry=stateTable[0][*source++]; - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - oredEntries|=entry=stateTable[0][*source++]; - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - oredEntries|=entry=stateTable[0][*source++]; - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - oredEntries|=entry=stateTable[0][*source++]; - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - oredEntries|=entry=stateTable[0][*source++]; - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - oredEntries|=entry=stateTable[0][*source++]; - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - oredEntries|=entry=stateTable[0][*source++]; - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - oredEntries|=entry=stateTable[0][*source++]; - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - oredEntries|=entry=stateTable[0][*source++]; - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - oredEntries|=entry=stateTable[0][*source++]; - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - oredEntries|=entry=stateTable[0][*source++]; - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - - /* were all 16 entries really valid? */ - if(!MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(oredEntries)) { - /* no, return to the first of these 16 */ - source-=16; - target-=16; - break; - } - } while(--count>0); - count=loops-count; - targetCapacity-=16*count; - - if(offsets!=NULL) { - lastSource+=16*count; - while(count>0) { - *offsets++=sourceIndex++; - *offsets++=sourceIndex++; - *offsets++=sourceIndex++; - *offsets++=sourceIndex++; - *offsets++=sourceIndex++; - *offsets++=sourceIndex++; - *offsets++=sourceIndex++; - *offsets++=sourceIndex++; - *offsets++=sourceIndex++; - *offsets++=sourceIndex++; - *offsets++=sourceIndex++; - *offsets++=sourceIndex++; - *offsets++=sourceIndex++; - *offsets++=sourceIndex++; - *offsets++=sourceIndex++; - *offsets++=sourceIndex++; - --count; - } - } - } -#endif - - /* conversion loop */ - while(targetCapacity > 0 && source < sourceLimit) { - entry=stateTable[0][*source++]; - /* MBCS_ENTRY_IS_FINAL(entry) */ - - /* test the most common case first */ - if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { - /* output BMP code point */ - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - --targetCapacity; - continue; - } - - /* - * An if-else-if chain provides more reliable performance for - * the most common cases compared to a switch. - */ - action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); - if(action==MBCS_STATE_FALLBACK_DIRECT_16) { - if(UCNV_TO_U_USE_FALLBACK(cnv)) { - /* output BMP code point */ - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - --targetCapacity; - continue; - } - } else if(action==MBCS_STATE_UNASSIGNED) { - /* just fall through */ - } else if(action==MBCS_STATE_ILLEGAL) { - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - } else { - /* reserved, must never occur */ - continue; - } - - /* set offsets since the start or the last extension */ - if(offsets!=NULL) { - int32_t count=(int32_t)(source-lastSource); - - /* predecrement: do not set the offset for the callback-causing character */ - while(--count>0) { - *offsets++=sourceIndex++; - } - /* offset and sourceIndex are now set for the current character */ - } - - if(U_FAILURE(*pErrorCode)) { - /* callback(illegal) */ - break; - } else /* unassigned sequences indicated with byteIndex>0 */ { - /* try an extension mapping */ - lastSource=source; - cnv->toUBytes[0]=*(source-1); - cnv->toULength=_extToU(cnv, cnv->sharedData, - 1, &source, sourceLimit, - &target, pArgs->targetLimit, - &offsets, sourceIndex, - pArgs->flush, - pErrorCode); - sourceIndex+=1+(int32_t)(source-lastSource); - - if(U_FAILURE(*pErrorCode)) { - /* not mappable or buffer overflow */ - break; - } - - /* recalculate the targetCapacity after an extension mapping */ - targetCapacity=(int32_t)(pArgs->targetLimit-target); - length=(int32_t)(sourceLimit-source); - if(length<targetCapacity) { - targetCapacity=length; - } - } - -#if MBCS_UNROLL_SINGLE_TO_BMP - /* unrolling makes it faster on Pentium III/Windows 2000 */ - goto unrolled; -#endif - } - - if(U_SUCCESS(*pErrorCode) && source<sourceLimit && target>=pArgs->targetLimit) { - /* target is full */ - *pErrorCode=U_BUFFER_OVERFLOW_ERROR; - } - - /* set offsets since the start or the last callback */ - if(offsets!=NULL) { - size_t count=source-lastSource; - while(count>0) { - *offsets++=sourceIndex++; - --count; - } - } - - /* write back the updated pointers */ - pArgs->source=(const char *)source; - pArgs->target=target; - pArgs->offsets=offsets; -} - -static UBool -hasValidTrailBytes(const int32_t (*stateTable)[256], uint8_t state) { - const int32_t *row=stateTable[state]; - int32_t b, entry; - /* First test for final entries in this state for some commonly valid byte values. */ - entry=row[0xa1]; - if( !MBCS_ENTRY_IS_TRANSITION(entry) && - MBCS_ENTRY_FINAL_ACTION(entry)!=MBCS_STATE_ILLEGAL - ) { - return TRUE; - } - entry=row[0x41]; - if( !MBCS_ENTRY_IS_TRANSITION(entry) && - MBCS_ENTRY_FINAL_ACTION(entry)!=MBCS_STATE_ILLEGAL - ) { - return TRUE; - } - /* Then test for final entries in this state. */ - for(b=0; b<=0xff; ++b) { - entry=row[b]; - if( !MBCS_ENTRY_IS_TRANSITION(entry) && - MBCS_ENTRY_FINAL_ACTION(entry)!=MBCS_STATE_ILLEGAL - ) { - return TRUE; - } - } - /* Then recurse for transition entries. */ - for(b=0; b<=0xff; ++b) { - entry=row[b]; - if( MBCS_ENTRY_IS_TRANSITION(entry) && - hasValidTrailBytes(stateTable, (uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry)) - ) { - return TRUE; - } - } - return FALSE; -} - -/* - * Is byte b a single/lead byte in this state? - * Recurse for transition states, because here we don't want to say that - * b is a lead byte if all byte sequences that start with b are illegal. - */ -static UBool -isSingleOrLead(const int32_t (*stateTable)[256], uint8_t state, UBool isDBCSOnly, uint8_t b) { - const int32_t *row=stateTable[state]; - int32_t entry=row[b]; - if(MBCS_ENTRY_IS_TRANSITION(entry)) { /* lead byte */ - return hasValidTrailBytes(stateTable, (uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry)); - } else { - uint8_t action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); - if(action==MBCS_STATE_CHANGE_ONLY && isDBCSOnly) { - return FALSE; /* SI/SO are illegal for DBCS-only conversion */ - } else { - return action!=MBCS_STATE_ILLEGAL; - } - } -} - -U_CFUNC void -ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, - UErrorCode *pErrorCode) { - UConverter *cnv; - const uint8_t *source, *sourceLimit; - UChar *target; - const UChar *targetLimit; - int32_t *offsets; - - const int32_t (*stateTable)[256]; - const uint16_t *unicodeCodeUnits; - - uint32_t offset; - uint8_t state; - int8_t byteIndex; - uint8_t *bytes; - - int32_t sourceIndex, nextSourceIndex; - - int32_t entry; - UChar c; - uint8_t action; - - /* use optimized function if possible */ - cnv=pArgs->converter; - - if(cnv->preToULength>0) { - /* - * pass sourceIndex=-1 because we continue from an earlier buffer - * in the future, this may change with continuous offsets - */ - ucnv_extContinueMatchToU(cnv, pArgs, -1, pErrorCode); - - if(U_FAILURE(*pErrorCode) || cnv->preToULength<0) { - return; - } - } - - if(cnv->sharedData->mbcs.countStates==1) { - if(!(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { - ucnv_MBCSSingleToBMPWithOffsets(pArgs, pErrorCode); - } else { - ucnv_MBCSSingleToUnicodeWithOffsets(pArgs, pErrorCode); - } - return; - } - - /* set up the local pointers */ - source=(const uint8_t *)pArgs->source; - sourceLimit=(const uint8_t *)pArgs->sourceLimit; - target=pArgs->target; - targetLimit=pArgs->targetLimit; - offsets=pArgs->offsets; - - if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { - stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; - } else { - stateTable=cnv->sharedData->mbcs.stateTable; - } - unicodeCodeUnits=cnv->sharedData->mbcs.unicodeCodeUnits; - - /* get the converter state from UConverter */ - offset=cnv->toUnicodeStatus; - byteIndex=cnv->toULength; - bytes=cnv->toUBytes; - - /* - * if we are in the SBCS state for a DBCS-only converter, - * then load the DBCS state from the MBCS data - * (dbcsOnlyState==0 if it is not a DBCS-only converter) - */ - if((state=(uint8_t)(cnv->mode))==0) { - state=cnv->sharedData->mbcs.dbcsOnlyState; - } - - /* sourceIndex=-1 if the current character began in the previous buffer */ - sourceIndex=byteIndex==0 ? 0 : -1; - nextSourceIndex=0; - - /* conversion loop */ - while(source<sourceLimit) { - /* - * This following test is to see if available input would overflow the output. - * It does not catch output of more than one code unit that - * overflows as a result of a surrogate pair or callback output - * from the last source byte. - * Therefore, those situations also test for overflows and will - * then break the loop, too. - */ - if(target>=targetLimit) { - /* target is full */ - *pErrorCode=U_BUFFER_OVERFLOW_ERROR; - break; - } - - if(byteIndex==0) { - /* optimized loop for 1/2-byte input and BMP output */ - if(offsets==NULL) { - do { - entry=stateTable[state][*source]; - if(MBCS_ENTRY_IS_TRANSITION(entry)) { - state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); - offset=MBCS_ENTRY_TRANSITION_OFFSET(entry); - - ++source; - if( source<sourceLimit && - MBCS_ENTRY_IS_FINAL(entry=stateTable[state][*source]) && - MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_VALID_16 && - (c=unicodeCodeUnits[offset+MBCS_ENTRY_FINAL_VALUE_16(entry)])<0xfffe - ) { - ++source; - *target++=c; - state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */ - offset=0; - } else { - /* set the state and leave the optimized loop */ - bytes[0]=*(source-1); - byteIndex=1; - break; - } - } else { - if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { - /* output BMP code point */ - ++source; - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */ - } else { - /* leave the optimized loop */ - break; - } - } - } while(source<sourceLimit && target<targetLimit); - } else /* offsets!=NULL */ { - do { - entry=stateTable[state][*source]; - if(MBCS_ENTRY_IS_TRANSITION(entry)) { - state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); - offset=MBCS_ENTRY_TRANSITION_OFFSET(entry); - - ++source; - if( source<sourceLimit && - MBCS_ENTRY_IS_FINAL(entry=stateTable[state][*source]) && - MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_VALID_16 && - (c=unicodeCodeUnits[offset+MBCS_ENTRY_FINAL_VALUE_16(entry)])<0xfffe - ) { - ++source; - *target++=c; - if(offsets!=NULL) { - *offsets++=sourceIndex; - sourceIndex=(nextSourceIndex+=2); - } - state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */ - offset=0; - } else { - /* set the state and leave the optimized loop */ - ++nextSourceIndex; - bytes[0]=*(source-1); - byteIndex=1; - break; - } - } else { - if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { - /* output BMP code point */ - ++source; - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - if(offsets!=NULL) { - *offsets++=sourceIndex; - sourceIndex=++nextSourceIndex; - } - state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */ - } else { - /* leave the optimized loop */ - break; - } - } - } while(source<sourceLimit && target<targetLimit); - } - - /* - * these tests and break statements could be put inside the loop - * if C had "break outerLoop" like Java - */ - if(source>=sourceLimit) { - break; - } - if(target>=targetLimit) { - /* target is full */ - *pErrorCode=U_BUFFER_OVERFLOW_ERROR; - break; - } - - ++nextSourceIndex; - bytes[byteIndex++]=*source++; - } else /* byteIndex>0 */ { - ++nextSourceIndex; - entry=stateTable[state][bytes[byteIndex++]=*source++]; - } - - if(MBCS_ENTRY_IS_TRANSITION(entry)) { - state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); - offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry); - continue; - } - - /* save the previous state for proper extension mapping with SI/SO-stateful converters */ - cnv->mode=state; - - /* set the next state early so that we can reuse the entry variable */ - state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */ - - /* - * An if-else-if chain provides more reliable performance for - * the most common cases compared to a switch. - */ - action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); - if(action==MBCS_STATE_VALID_16) { - offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); - c=unicodeCodeUnits[offset]; - if(c<0xfffe) { - /* output BMP code point */ - *target++=c; - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - byteIndex=0; - } else if(c==0xfffe) { - if(UCNV_TO_U_USE_FALLBACK(cnv) && (entry=(int32_t)ucnv_MBCSGetFallback(&cnv->sharedData->mbcs, offset))!=0xfffe) { - /* output fallback BMP code point */ - *target++=(UChar)entry; - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - byteIndex=0; - } - } else { - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - } - } else if(action==MBCS_STATE_VALID_DIRECT_16) { - /* output BMP code point */ - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - byteIndex=0; - } else if(action==MBCS_STATE_VALID_16_PAIR) { - offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); - c=unicodeCodeUnits[offset++]; - if(c<0xd800) { - /* output BMP code point below 0xd800 */ - *target++=c; - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - byteIndex=0; - } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? c<=0xdfff : c<=0xdbff) { - /* output roundtrip or fallback surrogate pair */ - *target++=(UChar)(c&0xdbff); - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - byteIndex=0; - if(target<targetLimit) { - *target++=unicodeCodeUnits[offset]; - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - } else { - /* target overflow */ - cnv->UCharErrorBuffer[0]=unicodeCodeUnits[offset]; - cnv->UCharErrorBufferLength=1; - *pErrorCode=U_BUFFER_OVERFLOW_ERROR; - - offset=0; - break; - } - } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? (c&0xfffe)==0xe000 : c==0xe000) { - /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */ - *target++=unicodeCodeUnits[offset]; - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - byteIndex=0; - } else if(c==0xffff) { - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - } - } else if(action==MBCS_STATE_VALID_DIRECT_20 || - (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv)) - ) { - entry=MBCS_ENTRY_FINAL_VALUE(entry); - /* output surrogate pair */ - *target++=(UChar)(0xd800|(UChar)(entry>>10)); - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - byteIndex=0; - c=(UChar)(0xdc00|(UChar)(entry&0x3ff)); - if(target<targetLimit) { - *target++=c; - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - } else { - /* target overflow */ - cnv->UCharErrorBuffer[0]=c; - cnv->UCharErrorBufferLength=1; - *pErrorCode=U_BUFFER_OVERFLOW_ERROR; - - offset=0; - break; - } - } else if(action==MBCS_STATE_CHANGE_ONLY) { - /* - * This serves as a state change without any output. - * It is useful for reading simple stateful encodings, - * for example using just Shift-In/Shift-Out codes. - * The 21 unused bits may later be used for more sophisticated - * state transitions. - */ - if(cnv->sharedData->mbcs.dbcsOnlyState==0) { - byteIndex=0; - } else { - /* SI/SO are illegal for DBCS-only conversion */ - state=(uint8_t)(cnv->mode); /* restore the previous state */ - - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - } - } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { - if(UCNV_TO_U_USE_FALLBACK(cnv)) { - /* output BMP code point */ - *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - byteIndex=0; - } - } else if(action==MBCS_STATE_UNASSIGNED) { - /* just fall through */ - } else if(action==MBCS_STATE_ILLEGAL) { - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - } else { - /* reserved, must never occur */ - byteIndex=0; - } - - /* end of action codes: prepare for a new character */ - offset=0; - - if(byteIndex==0) { - sourceIndex=nextSourceIndex; - } else if(U_FAILURE(*pErrorCode)) { - /* callback(illegal) */ - if(byteIndex>1) { - /* - * Ticket 5691: consistent illegal sequences: - * - We include at least the first byte in the illegal sequence. - * - If any of the non-initial bytes could be the start of a character, - * we stop the illegal sequence before the first one of those. - */ - UBool isDBCSOnly=(UBool)(cnv->sharedData->mbcs.dbcsOnlyState!=0); - int8_t i; - for(i=1; - i<byteIndex && !isSingleOrLead(stateTable, state, isDBCSOnly, bytes[i]); - ++i) {} - if(i<byteIndex) { - /* Back out some bytes. */ - int8_t backOutDistance=byteIndex-i; - int32_t bytesFromThisBuffer=(int32_t)(source-(const uint8_t *)pArgs->source); - byteIndex=i; /* length of reported illegal byte sequence */ - if(backOutDistance<=bytesFromThisBuffer) { - source-=backOutDistance; - } else { - /* Back out bytes from the previous buffer: Need to replay them. */ - cnv->preToULength=(int8_t)(bytesFromThisBuffer-backOutDistance); - /* preToULength is negative! */ - uprv_memcpy(cnv->preToU, bytes+i, -cnv->preToULength); - source=(const uint8_t *)pArgs->source; - } - } - } - break; - } else /* unassigned sequences indicated with byteIndex>0 */ { - /* try an extension mapping */ - pArgs->source=(const char *)source; - byteIndex=_extToU(cnv, cnv->sharedData, - byteIndex, &source, sourceLimit, - &target, targetLimit, - &offsets, sourceIndex, - pArgs->flush, - pErrorCode); - sourceIndex=nextSourceIndex+=(int32_t)(source-(const uint8_t *)pArgs->source); - - if(U_FAILURE(*pErrorCode)) { - /* not mappable or buffer overflow */ - break; - } - } - } - - /* set the converter state back into UConverter */ - cnv->toUnicodeStatus=offset; - cnv->mode=state; - cnv->toULength=byteIndex; - - /* write back the updated pointers */ - pArgs->source=(const char *)source; - pArgs->target=target; - pArgs->offsets=offsets; -} - -/* - * This version of ucnv_MBCSGetNextUChar() is optimized for single-byte, single-state codepages. - * We still need a conversion loop in case we find reserved action codes, which are to be ignored. - */ -static UChar32 -ucnv_MBCSSingleGetNextUChar(UConverterToUnicodeArgs *pArgs, - UErrorCode *pErrorCode) { - UConverter *cnv; - const int32_t (*stateTable)[256]; - const uint8_t *source, *sourceLimit; - - int32_t entry; - uint8_t action; - - /* set up the local pointers */ - cnv=pArgs->converter; - source=(const uint8_t *)pArgs->source; - sourceLimit=(const uint8_t *)pArgs->sourceLimit; - if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { - stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; - } else { - stateTable=cnv->sharedData->mbcs.stateTable; - } - - /* conversion loop */ - while(source<sourceLimit) { - entry=stateTable[0][*source++]; - /* MBCS_ENTRY_IS_FINAL(entry) */ - - /* write back the updated pointer early so that we can return directly */ - pArgs->source=(const char *)source; - - if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { - /* output BMP code point */ - return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - } - - /* - * An if-else-if chain provides more reliable performance for - * the most common cases compared to a switch. - */ - action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); - if( action==MBCS_STATE_VALID_DIRECT_20 || - (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv)) - ) { - /* output supplementary code point */ - return (UChar32)(MBCS_ENTRY_FINAL_VALUE(entry)+0x10000); - } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { - if(UCNV_TO_U_USE_FALLBACK(cnv)) { - /* output BMP code point */ - return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - } - } else if(action==MBCS_STATE_UNASSIGNED) { - /* just fall through */ - } else if(action==MBCS_STATE_ILLEGAL) { - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - } else { - /* reserved, must never occur */ - continue; - } - - if(U_FAILURE(*pErrorCode)) { - /* callback(illegal) */ - break; - } else /* unassigned sequence */ { - /* defer to the generic implementation */ - pArgs->source=(const char *)source-1; - return UCNV_GET_NEXT_UCHAR_USE_TO_U; - } - } - - /* no output because of empty input or only state changes */ - *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; - return 0xffff; -} - -/* - * Version of _MBCSToUnicodeWithOffsets() optimized for single-character - * conversion without offset handling. - * - * When a character does not have a mapping to Unicode, then we return to the - * generic ucnv_getNextUChar() code for extension/GB 18030 and error/callback - * handling. - * We also defer to the generic code in other complicated cases and have them - * ultimately handled by _MBCSToUnicodeWithOffsets() itself. - * - * All normal mappings and errors are handled here. - */ -static UChar32 U_CALLCONV -ucnv_MBCSGetNextUChar(UConverterToUnicodeArgs *pArgs, - UErrorCode *pErrorCode) { - UConverter *cnv; - const uint8_t *source, *sourceLimit, *lastSource; - - const int32_t (*stateTable)[256]; - const uint16_t *unicodeCodeUnits; - - uint32_t offset; - uint8_t state; - - int32_t entry; - UChar32 c; - uint8_t action; - - /* use optimized function if possible */ - cnv=pArgs->converter; - - if(cnv->preToULength>0) { - /* use the generic code in ucnv_getNextUChar() to continue with a partial match */ - return UCNV_GET_NEXT_UCHAR_USE_TO_U; - } - - if(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SURROGATES) { - /* - * Using the generic ucnv_getNextUChar() code lets us deal correctly - * with the rare case of a codepage that maps single surrogates - * without adding the complexity to this already complicated function here. - */ - return UCNV_GET_NEXT_UCHAR_USE_TO_U; - } else if(cnv->sharedData->mbcs.countStates==1) { - return ucnv_MBCSSingleGetNextUChar(pArgs, pErrorCode); - } - - /* set up the local pointers */ - source=lastSource=(const uint8_t *)pArgs->source; - sourceLimit=(const uint8_t *)pArgs->sourceLimit; - - if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { - stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; - } else { - stateTable=cnv->sharedData->mbcs.stateTable; - } - unicodeCodeUnits=cnv->sharedData->mbcs.unicodeCodeUnits; - - /* get the converter state from UConverter */ - offset=cnv->toUnicodeStatus; - - /* - * if we are in the SBCS state for a DBCS-only converter, - * then load the DBCS state from the MBCS data - * (dbcsOnlyState==0 if it is not a DBCS-only converter) - */ - if((state=(uint8_t)(cnv->mode))==0) { - state=cnv->sharedData->mbcs.dbcsOnlyState; - } - - /* conversion loop */ - c=U_SENTINEL; - while(source<sourceLimit) { - entry=stateTable[state][*source++]; - if(MBCS_ENTRY_IS_TRANSITION(entry)) { - state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); - offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry); - - /* optimization for 1/2-byte input and BMP output */ - if( source<sourceLimit && - MBCS_ENTRY_IS_FINAL(entry=stateTable[state][*source]) && - MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_VALID_16 && - (c=unicodeCodeUnits[offset+MBCS_ENTRY_FINAL_VALUE_16(entry)])<0xfffe - ) { - ++source; - state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */ - /* output BMP code point */ - break; - } - } else { - /* save the previous state for proper extension mapping with SI/SO-stateful converters */ - cnv->mode=state; - - /* set the next state early so that we can reuse the entry variable */ - state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */ - - /* - * An if-else-if chain provides more reliable performance for - * the most common cases compared to a switch. - */ - action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); - if(action==MBCS_STATE_VALID_DIRECT_16) { - /* output BMP code point */ - c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - break; - } else if(action==MBCS_STATE_VALID_16) { - offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); - c=unicodeCodeUnits[offset]; - if(c<0xfffe) { - /* output BMP code point */ - break; - } else if(c==0xfffe) { - if(UCNV_TO_U_USE_FALLBACK(cnv) && (c=ucnv_MBCSGetFallback(&cnv->sharedData->mbcs, offset))!=0xfffe) { - break; - } - } else { - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - } - } else if(action==MBCS_STATE_VALID_16_PAIR) { - offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); - c=unicodeCodeUnits[offset++]; - if(c<0xd800) { - /* output BMP code point below 0xd800 */ - break; - } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? c<=0xdfff : c<=0xdbff) { - /* output roundtrip or fallback supplementary code point */ - c=((c&0x3ff)<<10)+unicodeCodeUnits[offset]+(0x10000-0xdc00); - break; - } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? (c&0xfffe)==0xe000 : c==0xe000) { - /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */ - c=unicodeCodeUnits[offset]; - break; - } else if(c==0xffff) { - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - } - } else if(action==MBCS_STATE_VALID_DIRECT_20 || - (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv)) - ) { - /* output supplementary code point */ - c=(UChar32)(MBCS_ENTRY_FINAL_VALUE(entry)+0x10000); - break; - } else if(action==MBCS_STATE_CHANGE_ONLY) { - /* - * This serves as a state change without any output. - * It is useful for reading simple stateful encodings, - * for example using just Shift-In/Shift-Out codes. - * The 21 unused bits may later be used for more sophisticated - * state transitions. - */ - if(cnv->sharedData->mbcs.dbcsOnlyState!=0) { - /* SI/SO are illegal for DBCS-only conversion */ - state=(uint8_t)(cnv->mode); /* restore the previous state */ - - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - } - } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { - if(UCNV_TO_U_USE_FALLBACK(cnv)) { - /* output BMP code point */ - c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - break; - } - } else if(action==MBCS_STATE_UNASSIGNED) { - /* just fall through */ - } else if(action==MBCS_STATE_ILLEGAL) { - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - } else { - /* reserved (must never occur), or only state change */ - offset=0; - lastSource=source; - continue; - } - - /* end of action codes: prepare for a new character */ - offset=0; - - if(U_FAILURE(*pErrorCode)) { - /* callback(illegal) */ - break; - } else /* unassigned sequence */ { - /* defer to the generic implementation */ - cnv->toUnicodeStatus=0; - cnv->mode=state; - pArgs->source=(const char *)lastSource; - return UCNV_GET_NEXT_UCHAR_USE_TO_U; - } - } - } - - if(c<0) { - if(U_SUCCESS(*pErrorCode) && source==sourceLimit && lastSource<source) { - /* incomplete character byte sequence */ - uint8_t *bytes=cnv->toUBytes; - cnv->toULength=(int8_t)(source-lastSource); - do { - *bytes++=*lastSource++; - } while(lastSource<source); - *pErrorCode=U_TRUNCATED_CHAR_FOUND; - } else if(U_FAILURE(*pErrorCode)) { - /* callback(illegal) */ - /* - * Ticket 5691: consistent illegal sequences: - * - We include at least the first byte in the illegal sequence. - * - If any of the non-initial bytes could be the start of a character, - * we stop the illegal sequence before the first one of those. - */ - UBool isDBCSOnly=(UBool)(cnv->sharedData->mbcs.dbcsOnlyState!=0); - uint8_t *bytes=cnv->toUBytes; - *bytes++=*lastSource++; /* first byte */ - if(lastSource==source) { - cnv->toULength=1; - } else /* lastSource<source: multi-byte character */ { - int8_t i; - for(i=1; - lastSource<source && !isSingleOrLead(stateTable, state, isDBCSOnly, *lastSource); - ++i - ) { - *bytes++=*lastSource++; - } - cnv->toULength=i; - source=lastSource; - } - } else { - /* no output because of empty input or only state changes */ - *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; - } - c=0xffff; - } - - /* set the converter state back into UConverter, ready for a new character */ - cnv->toUnicodeStatus=0; - cnv->mode=state; - - /* write back the updated pointer */ - pArgs->source=(const char *)source; - return c; -} - -#if 0 -/* - * Code disabled 2002dec09 (ICU 2.4) because it is not currently used in ICU. markus - * Removal improves code coverage. - */ -/** - * This version of ucnv_MBCSSimpleGetNextUChar() is optimized for single-byte, single-state codepages. - * It does not handle the EBCDIC swaplfnl option (set in UConverter). - * It does not handle conversion extensions (_extToU()). - */ -U_CFUNC UChar32 -ucnv_MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData, - uint8_t b, UBool useFallback) { - int32_t entry; - uint8_t action; - - entry=sharedData->mbcs.stateTable[0][b]; - /* MBCS_ENTRY_IS_FINAL(entry) */ - - if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { - /* output BMP code point */ - return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - } - - /* - * An if-else-if chain provides more reliable performance for - * the most common cases compared to a switch. - */ - action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); - if(action==MBCS_STATE_VALID_DIRECT_20) { - /* output supplementary code point */ - return 0x10000+MBCS_ENTRY_FINAL_VALUE(entry); - } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { - if(!TO_U_USE_FALLBACK(useFallback)) { - return 0xfffe; - } - /* output BMP code point */ - return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - } else if(action==MBCS_STATE_FALLBACK_DIRECT_20) { - if(!TO_U_USE_FALLBACK(useFallback)) { - return 0xfffe; - } - /* output supplementary code point */ - return 0x10000+MBCS_ENTRY_FINAL_VALUE(entry); - } else if(action==MBCS_STATE_UNASSIGNED) { - return 0xfffe; - } else if(action==MBCS_STATE_ILLEGAL) { - return 0xffff; - } else { - /* reserved, must never occur */ - return 0xffff; - } -} -#endif - -/* - * This is a simple version of _MBCSGetNextUChar() that is used - * by other converter implementations. - * It only returns an "assigned" result if it consumes the entire input. - * It does not use state from the converter, nor error codes. - * It does not handle the EBCDIC swaplfnl option (set in UConverter). - * It handles conversion extensions but not GB 18030. - * - * Return value: - * U+fffe unassigned - * U+ffff illegal - * otherwise the Unicode code point - */ -U_CFUNC UChar32 -ucnv_MBCSSimpleGetNextUChar(UConverterSharedData *sharedData, - const char *source, int32_t length, - UBool useFallback) { - const int32_t (*stateTable)[256]; - const uint16_t *unicodeCodeUnits; - - uint32_t offset; - uint8_t state, action; - - UChar32 c; - int32_t i, entry; - - if(length<=0) { - /* no input at all: "illegal" */ - return 0xffff; - } - -#if 0 -/* - * Code disabled 2002dec09 (ICU 2.4) because it is not currently used in ICU. markus - * TODO In future releases, verify that this function is never called for SBCS - * conversions, i.e., that sharedData->mbcs.countStates==1 is still true. - * Removal improves code coverage. - */ - /* use optimized function if possible */ - if(sharedData->mbcs.countStates==1) { - if(length==1) { - return ucnv_MBCSSingleSimpleGetNextUChar(sharedData, (uint8_t)*source, useFallback); - } else { - return 0xffff; /* illegal: more than a single byte for an SBCS converter */ - } - } -#endif - - /* set up the local pointers */ - stateTable=sharedData->mbcs.stateTable; - unicodeCodeUnits=sharedData->mbcs.unicodeCodeUnits; - - /* converter state */ - offset=0; - state=sharedData->mbcs.dbcsOnlyState; - - /* conversion loop */ - for(i=0;;) { - entry=stateTable[state][(uint8_t)source[i++]]; - if(MBCS_ENTRY_IS_TRANSITION(entry)) { - state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); - offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry); - - if(i==length) { - return 0xffff; /* truncated character */ - } - } else { - /* - * An if-else-if chain provides more reliable performance for - * the most common cases compared to a switch. - */ - action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); - if(action==MBCS_STATE_VALID_16) { - offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); - c=unicodeCodeUnits[offset]; - if(c!=0xfffe) { - /* done */ - } else if(UCNV_TO_U_USE_FALLBACK(cnv)) { - c=ucnv_MBCSGetFallback(&sharedData->mbcs, offset); - /* else done with 0xfffe */ - } - break; - } else if(action==MBCS_STATE_VALID_DIRECT_16) { - /* output BMP code point */ - c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - break; - } else if(action==MBCS_STATE_VALID_16_PAIR) { - offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); - c=unicodeCodeUnits[offset++]; - if(c<0xd800) { - /* output BMP code point below 0xd800 */ - } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? c<=0xdfff : c<=0xdbff) { - /* output roundtrip or fallback supplementary code point */ - c=(UChar32)(((c&0x3ff)<<10)+unicodeCodeUnits[offset]+(0x10000-0xdc00)); - } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? (c&0xfffe)==0xe000 : c==0xe000) { - /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */ - c=unicodeCodeUnits[offset]; - } else if(c==0xffff) { - return 0xffff; - } else { - c=0xfffe; - } - break; - } else if(action==MBCS_STATE_VALID_DIRECT_20) { - /* output supplementary code point */ - c=0x10000+MBCS_ENTRY_FINAL_VALUE(entry); - break; - } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { - if(!TO_U_USE_FALLBACK(useFallback)) { - c=0xfffe; - break; - } - /* output BMP code point */ - c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); - break; - } else if(action==MBCS_STATE_FALLBACK_DIRECT_20) { - if(!TO_U_USE_FALLBACK(useFallback)) { - c=0xfffe; - break; - } - /* output supplementary code point */ - c=0x10000+MBCS_ENTRY_FINAL_VALUE(entry); - break; - } else if(action==MBCS_STATE_UNASSIGNED) { - c=0xfffe; - break; - } - - /* - * forbid MBCS_STATE_CHANGE_ONLY for this function, - * and MBCS_STATE_ILLEGAL and reserved action codes - */ - return 0xffff; - } - } - - if(i!=length) { - /* illegal for this function: not all input consumed */ - return 0xffff; - } - - if(c==0xfffe) { - /* try an extension mapping */ - const int32_t *cx=sharedData->mbcs.extIndexes; - if(cx!=NULL) { - return ucnv_extSimpleMatchToU(cx, source, length, useFallback); - } - } - - return c; -} - -/* MBCS-from-Unicode conversion functions ----------------------------------- */ - -/* This version of ucnv_MBCSFromUnicodeWithOffsets() is optimized for double-byte codepages. */ -static void -ucnv_MBCSDoubleFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, - UErrorCode *pErrorCode) { - UConverter *cnv; - const UChar *source, *sourceLimit; - uint8_t *target; - int32_t targetCapacity; - int32_t *offsets; - - const uint16_t *table; - const uint16_t *mbcsIndex; - const uint8_t *bytes; - - UChar32 c; - - int32_t sourceIndex, nextSourceIndex; - - uint32_t stage2Entry; - uint32_t asciiRoundtrips; - uint32_t value; - uint8_t unicodeMask; - - /* use optimized function if possible */ - cnv=pArgs->converter; - unicodeMask=cnv->sharedData->mbcs.unicodeMask; - - /* set up the local pointers */ - source=pArgs->source; - sourceLimit=pArgs->sourceLimit; - target=(uint8_t *)pArgs->target; - targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); - offsets=pArgs->offsets; - - table=cnv->sharedData->mbcs.fromUnicodeTable; - mbcsIndex=cnv->sharedData->mbcs.mbcsIndex; - if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { - bytes=cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; - } else { - bytes=cnv->sharedData->mbcs.fromUnicodeBytes; - } - asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; - - /* get the converter state from UConverter */ - c=cnv->fromUChar32; - - /* sourceIndex=-1 if the current character began in the previous buffer */ - sourceIndex= c==0 ? 0 : -1; - nextSourceIndex=0; - - /* conversion loop */ - if(c!=0 && targetCapacity>0) { - goto getTrail; - } - - while(source<sourceLimit) { - /* - * This following test is to see if available input would overflow the output. - * It does not catch output of more than one byte that - * overflows as a result of a multi-byte character or callback output - * from the last source character. - * Therefore, those situations also test for overflows and will - * then break the loop, too. - */ - if(targetCapacity>0) { - /* - * Get a correct Unicode code point: - * a single UChar for a BMP code point or - * a matched surrogate pair for a "supplementary code point". - */ - c=*source++; - ++nextSourceIndex; - if(c<=0x7f && IS_ASCII_ROUNDTRIP(c, asciiRoundtrips)) { - *target++=(uint8_t)c; - if(offsets!=NULL) { - *offsets++=sourceIndex; - sourceIndex=nextSourceIndex; - } - --targetCapacity; - c=0; - continue; - } - /* - * utf8Friendly table: Test for <=0xd7ff rather than <=MBCS_FAST_MAX - * to avoid dealing with surrogates. - * MBCS_FAST_MAX must be >=0xd7ff. - */ - if(c<=0xd7ff) { - value=DBCS_RESULT_FROM_MOST_BMP(mbcsIndex, (const uint16_t *)bytes, c); - /* There are only roundtrips (!=0) and no-mapping (==0) entries. */ - if(value==0) { - goto unassigned; - } - /* output the value */ - } else { - /* - * This also tests if the codepage maps single surrogates. - * If it does, then surrogates are not paired but mapped separately. - * Note that in this case unmatched surrogates are not detected. - */ - if(U16_IS_SURROGATE(c) && !(unicodeMask&UCNV_HAS_SURROGATES)) { - if(U16_IS_SURROGATE_LEAD(c)) { -getTrail: - if(source<sourceLimit) { - /* test the following code unit */ - UChar trail=*source; - if(U16_IS_TRAIL(trail)) { - ++source; - ++nextSourceIndex; - c=U16_GET_SUPPLEMENTARY(c, trail); - if(!(unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { - /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ - /* callback(unassigned) */ - goto unassigned; - } - /* convert this supplementary code point */ - /* exit this condition tree */ - } else { - /* this is an unmatched lead code unit (1st surrogate) */ - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - break; - } - } else { - /* no more input */ - break; - } - } else { - /* this is an unmatched trail code unit (2nd surrogate) */ - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - break; - } - } - - /* convert the Unicode code point in c into codepage bytes */ - stage2Entry=MBCS_STAGE_2_FROM_U(table, c); - - /* get the bytes and the length for the output */ - /* MBCS_OUTPUT_2 */ - value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c); - - /* is this code point assigned, or do we use fallbacks? */ - if(!(MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) || - (UCNV_FROM_U_USE_FALLBACK(cnv, c) && value!=0)) - ) { - /* - * We allow a 0 byte output if the "assigned" bit is set for this entry. - * There is no way with this data structure for fallback output - * to be a zero byte. - */ - -unassigned: - /* try an extension mapping */ - pArgs->source=source; - c=_extFromU(cnv, cnv->sharedData, - c, &source, sourceLimit, - &target, target+targetCapacity, - &offsets, sourceIndex, - pArgs->flush, - pErrorCode); - nextSourceIndex+=(int32_t)(source-pArgs->source); - - if(U_FAILURE(*pErrorCode)) { - /* not mappable or buffer overflow */ - break; - } else { - /* a mapping was written to the target, continue */ - - /* recalculate the targetCapacity after an extension mapping */ - targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target); - - /* normal end of conversion: prepare for a new character */ - sourceIndex=nextSourceIndex; - continue; - } - } - } - - /* write the output character bytes from value and length */ - /* from the first if in the loop we know that targetCapacity>0 */ - if(value<=0xff) { - /* this is easy because we know that there is enough space */ - *target++=(uint8_t)value; - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - --targetCapacity; - } else /* length==2 */ { - *target++=(uint8_t)(value>>8); - if(2<=targetCapacity) { - *target++=(uint8_t)value; - if(offsets!=NULL) { - *offsets++=sourceIndex; - *offsets++=sourceIndex; - } - targetCapacity-=2; - } else { - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - cnv->charErrorBuffer[0]=(char)value; - cnv->charErrorBufferLength=1; - - /* target overflow */ - targetCapacity=0; - *pErrorCode=U_BUFFER_OVERFLOW_ERROR; - c=0; - break; - } - } - - /* normal end of conversion: prepare for a new character */ - c=0; - sourceIndex=nextSourceIndex; - continue; - } else { - /* target is full */ - *pErrorCode=U_BUFFER_OVERFLOW_ERROR; - break; - } - } - - /* set the converter state back into UConverter */ - cnv->fromUChar32=c; - - /* write back the updated pointers */ - pArgs->source=source; - pArgs->target=(char *)target; - pArgs->offsets=offsets; -} - -/* This version of ucnv_MBCSFromUnicodeWithOffsets() is optimized for single-byte codepages. */ -static void -ucnv_MBCSSingleFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, - UErrorCode *pErrorCode) { - UConverter *cnv; - const UChar *source, *sourceLimit; - uint8_t *target; - int32_t targetCapacity; - int32_t *offsets; - - const uint16_t *table; - const uint16_t *results; - - UChar32 c; - - int32_t sourceIndex, nextSourceIndex; - - uint16_t value, minValue; - UBool hasSupplementary; - - /* set up the local pointers */ - cnv=pArgs->converter; - source=pArgs->source; - sourceLimit=pArgs->sourceLimit; - target=(uint8_t *)pArgs->target; - targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); - offsets=pArgs->offsets; - - table=cnv->sharedData->mbcs.fromUnicodeTable; - if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { - results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; - } else { - results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes; - } - - if(cnv->useFallback) { - /* use all roundtrip and fallback results */ - minValue=0x800; - } else { - /* use only roundtrips and fallbacks from private-use characters */ - minValue=0xc00; - } - hasSupplementary=(UBool)(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY); - - /* get the converter state from UConverter */ - c=cnv->fromUChar32; - - /* sourceIndex=-1 if the current character began in the previous buffer */ - sourceIndex= c==0 ? 0 : -1; - nextSourceIndex=0; - - /* conversion loop */ - if(c!=0 && targetCapacity>0) { - goto getTrail; - } - - while(source<sourceLimit) { - /* - * This following test is to see if available input would overflow the output. - * It does not catch output of more than one byte that - * overflows as a result of a multi-byte character or callback output - * from the last source character. - * Therefore, those situations also test for overflows and will - * then break the loop, too. - */ - if(targetCapacity>0) { - /* - * Get a correct Unicode code point: - * a single UChar for a BMP code point or - * a matched surrogate pair for a "supplementary code point". - */ - c=*source++; - ++nextSourceIndex; - if(U16_IS_SURROGATE(c)) { - if(U16_IS_SURROGATE_LEAD(c)) { -getTrail: - if(source<sourceLimit) { - /* test the following code unit */ - UChar trail=*source; - if(U16_IS_TRAIL(trail)) { - ++source; - ++nextSourceIndex; - c=U16_GET_SUPPLEMENTARY(c, trail); - if(!hasSupplementary) { - /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ - /* callback(unassigned) */ - goto unassigned; - } - /* convert this supplementary code point */ - /* exit this condition tree */ - } else { - /* this is an unmatched lead code unit (1st surrogate) */ - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - break; - } - } else { - /* no more input */ - break; - } - } else { - /* this is an unmatched trail code unit (2nd surrogate) */ - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - break; - } - } - - /* convert the Unicode code point in c into codepage bytes */ - value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); - - /* is this code point assigned, or do we use fallbacks? */ - if(value>=minValue) { - /* assigned, write the output character bytes from value and length */ - /* length==1 */ - /* this is easy because we know that there is enough space */ - *target++=(uint8_t)value; - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - --targetCapacity; - - /* normal end of conversion: prepare for a new character */ - c=0; - sourceIndex=nextSourceIndex; - } else { /* unassigned */ -unassigned: - /* try an extension mapping */ - pArgs->source=source; - c=_extFromU(cnv, cnv->sharedData, - c, &source, sourceLimit, - &target, target+targetCapacity, - &offsets, sourceIndex, - pArgs->flush, - pErrorCode); - nextSourceIndex+=(int32_t)(source-pArgs->source); - - if(U_FAILURE(*pErrorCode)) { - /* not mappable or buffer overflow */ - break; - } else { - /* a mapping was written to the target, continue */ - - /* recalculate the targetCapacity after an extension mapping */ - targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target); - - /* normal end of conversion: prepare for a new character */ - sourceIndex=nextSourceIndex; - } - } - } else { - /* target is full */ - *pErrorCode=U_BUFFER_OVERFLOW_ERROR; - break; - } - } - - /* set the converter state back into UConverter */ - cnv->fromUChar32=c; - - /* write back the updated pointers */ - pArgs->source=source; - pArgs->target=(char *)target; - pArgs->offsets=offsets; -} - -/* - * This version of ucnv_MBCSFromUnicode() is optimized for single-byte codepages - * that map only to and from the BMP. - * In addition to single-byte/state optimizations, the offset calculations - * become much easier. - * It would be possible to use the sbcsIndex for UTF-8-friendly tables, - * but measurements have shown that this diminishes performance - * in more cases than it improves it. - * See SVN revision 21013 (2007-feb-06) for the last version with #if switches - * for various MBCS and SBCS optimizations. - */ -static void -ucnv_MBCSSingleFromBMPWithOffsets(UConverterFromUnicodeArgs *pArgs, - UErrorCode *pErrorCode) { - UConverter *cnv; - const UChar *source, *sourceLimit, *lastSource; - uint8_t *target; - int32_t targetCapacity, length; - int32_t *offsets; - - const uint16_t *table; - const uint16_t *results; - - UChar32 c; - - int32_t sourceIndex; - - uint32_t asciiRoundtrips; - uint16_t value, minValue; - - /* set up the local pointers */ - cnv=pArgs->converter; - source=pArgs->source; - sourceLimit=pArgs->sourceLimit; - target=(uint8_t *)pArgs->target; - targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); - offsets=pArgs->offsets; - - table=cnv->sharedData->mbcs.fromUnicodeTable; - if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { - results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; - } else { - results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes; - } - asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; - - if(cnv->useFallback) { - /* use all roundtrip and fallback results */ - minValue=0x800; - } else { - /* use only roundtrips and fallbacks from private-use characters */ - minValue=0xc00; - } - - /* get the converter state from UConverter */ - c=cnv->fromUChar32; - - /* sourceIndex=-1 if the current character began in the previous buffer */ - sourceIndex= c==0 ? 0 : -1; - lastSource=source; - - /* - * since the conversion here is 1:1 UChar:uint8_t, we need only one counter - * for the minimum of the sourceLength and targetCapacity - */ - length=(int32_t)(sourceLimit-source); - if(length<targetCapacity) { - targetCapacity=length; - } - - /* conversion loop */ - if(c!=0 && targetCapacity>0) { - goto getTrail; - } - -#if MBCS_UNROLL_SINGLE_FROM_BMP - /* unrolling makes it slower on Pentium III/Windows 2000?! */ - /* unroll the loop with the most common case */ -unrolled: - if(targetCapacity>=4) { - int32_t count, loops; - uint16_t andedValues; - - loops=count=targetCapacity>>2; - do { - c=*source++; - andedValues=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); - *target++=(uint8_t)value; - c=*source++; - andedValues&=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); - *target++=(uint8_t)value; - c=*source++; - andedValues&=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); - *target++=(uint8_t)value; - c=*source++; - andedValues&=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); - *target++=(uint8_t)value; - - /* were all 4 entries really valid? */ - if(andedValues<minValue) { - /* no, return to the first of these 4 */ - source-=4; - target-=4; - break; - } - } while(--count>0); - count=loops-count; - targetCapacity-=4*count; - - if(offsets!=NULL) { - lastSource+=4*count; - while(count>0) { - *offsets++=sourceIndex++; - *offsets++=sourceIndex++; - *offsets++=sourceIndex++; - *offsets++=sourceIndex++; - --count; - } - } - - c=0; - } -#endif - - while(targetCapacity>0) { - /* - * Get a correct Unicode code point: - * a single UChar for a BMP code point or - * a matched surrogate pair for a "supplementary code point". - */ - c=*source++; - /* - * Do not immediately check for single surrogates: - * Assume that they are unassigned and check for them in that case. - * This speeds up the conversion of assigned characters. - */ - /* convert the Unicode code point in c into codepage bytes */ - if(c<=0x7f && IS_ASCII_ROUNDTRIP(c, asciiRoundtrips)) { - *target++=(uint8_t)c; - --targetCapacity; - c=0; - continue; - } - value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); - /* is this code point assigned, or do we use fallbacks? */ - if(value>=minValue) { - /* assigned, write the output character bytes from value and length */ - /* length==1 */ - /* this is easy because we know that there is enough space */ - *target++=(uint8_t)value; - --targetCapacity; - - /* normal end of conversion: prepare for a new character */ - c=0; - continue; - } else if(!U16_IS_SURROGATE(c)) { - /* normal, unassigned BMP character */ - } else if(U16_IS_SURROGATE_LEAD(c)) { -getTrail: - if(source<sourceLimit) { - /* test the following code unit */ - UChar trail=*source; - if(U16_IS_TRAIL(trail)) { - ++source; - c=U16_GET_SUPPLEMENTARY(c, trail); - /* this codepage does not map supplementary code points */ - /* callback(unassigned) */ - } else { - /* this is an unmatched lead code unit (1st surrogate) */ - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - break; - } - } else { - /* no more input */ - if (pArgs->flush) { - *pErrorCode=U_TRUNCATED_CHAR_FOUND; - } - break; - } - } else { - /* this is an unmatched trail code unit (2nd surrogate) */ - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - break; - } - - /* c does not have a mapping */ - - /* get the number of code units for c to correctly advance sourceIndex */ - length=U16_LENGTH(c); - - /* set offsets since the start or the last extension */ - if(offsets!=NULL) { - int32_t count=(int32_t)(source-lastSource); - - /* do not set the offset for this character */ - count-=length; - - while(count>0) { - *offsets++=sourceIndex++; - --count; - } - /* offsets and sourceIndex are now set for the current character */ - } - - /* try an extension mapping */ - lastSource=source; - c=_extFromU(cnv, cnv->sharedData, - c, &source, sourceLimit, - &target, (const uint8_t *)(pArgs->targetLimit), - &offsets, sourceIndex, - pArgs->flush, - pErrorCode); - sourceIndex+=length+(int32_t)(source-lastSource); - lastSource=source; - - if(U_FAILURE(*pErrorCode)) { - /* not mappable or buffer overflow */ - break; - } else { - /* a mapping was written to the target, continue */ - - /* recalculate the targetCapacity after an extension mapping */ - targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target); - length=(int32_t)(sourceLimit-source); - if(length<targetCapacity) { - targetCapacity=length; - } - } - -#if MBCS_UNROLL_SINGLE_FROM_BMP - /* unrolling makes it slower on Pentium III/Windows 2000?! */ - goto unrolled; -#endif - } - - if(U_SUCCESS(*pErrorCode) && source<sourceLimit && target>=(uint8_t *)pArgs->targetLimit) { - /* target is full */ - *pErrorCode=U_BUFFER_OVERFLOW_ERROR; - } - - /* set offsets since the start or the last callback */ - if(offsets!=NULL) { - size_t count=source-lastSource; - if (count > 0 && *pErrorCode == U_TRUNCATED_CHAR_FOUND) { - /* - Caller gave us a partial supplementary character, - which this function couldn't convert in any case. - The callback will handle the offset. - */ - count--; - } - while(count>0) { - *offsets++=sourceIndex++; - --count; - } - } - - /* set the converter state back into UConverter */ - cnv->fromUChar32=c; - - /* write back the updated pointers */ - pArgs->source=source; - pArgs->target=(char *)target; - pArgs->offsets=offsets; -} - -U_CFUNC void -ucnv_MBCSFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, - UErrorCode *pErrorCode) { - UConverter *cnv; - const UChar *source, *sourceLimit; - uint8_t *target; - int32_t targetCapacity; - int32_t *offsets; - - const uint16_t *table; - const uint16_t *mbcsIndex; - const uint8_t *p, *bytes; - uint8_t outputType; - - UChar32 c; - - int32_t prevSourceIndex, sourceIndex, nextSourceIndex; - - uint32_t stage2Entry; - uint32_t asciiRoundtrips; - uint32_t value; - /* Shift-In and Shift-Out byte sequences differ by encoding scheme. */ - uint8_t siBytes[2] = {0, 0}; - uint8_t soBytes[2] = {0, 0}; - uint8_t siLength, soLength; - int32_t length = 0, prevLength; - uint8_t unicodeMask; - - cnv=pArgs->converter; - - if(cnv->preFromUFirstCP>=0) { - /* - * pass sourceIndex=-1 because we continue from an earlier buffer - * in the future, this may change with continuous offsets - */ - ucnv_extContinueMatchFromU(cnv, pArgs, -1, pErrorCode); - - if(U_FAILURE(*pErrorCode) || cnv->preFromULength<0) { - return; - } - } - - /* use optimized function if possible */ - outputType=cnv->sharedData->mbcs.outputType; - unicodeMask=cnv->sharedData->mbcs.unicodeMask; - if(outputType==MBCS_OUTPUT_1 && !(unicodeMask&UCNV_HAS_SURROGATES)) { - if(!(unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { - ucnv_MBCSSingleFromBMPWithOffsets(pArgs, pErrorCode); - } else { - ucnv_MBCSSingleFromUnicodeWithOffsets(pArgs, pErrorCode); - } - return; - } else if(outputType==MBCS_OUTPUT_2 && cnv->sharedData->mbcs.utf8Friendly) { - ucnv_MBCSDoubleFromUnicodeWithOffsets(pArgs, pErrorCode); - return; - } - - /* set up the local pointers */ - source=pArgs->source; - sourceLimit=pArgs->sourceLimit; - target=(uint8_t *)pArgs->target; - targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); - offsets=pArgs->offsets; - - table=cnv->sharedData->mbcs.fromUnicodeTable; - if(cnv->sharedData->mbcs.utf8Friendly) { - mbcsIndex=cnv->sharedData->mbcs.mbcsIndex; - } else { - mbcsIndex=NULL; - } - if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { - bytes=cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; - } else { - bytes=cnv->sharedData->mbcs.fromUnicodeBytes; - } - asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; - - /* get the converter state from UConverter */ - c=cnv->fromUChar32; - - if(outputType==MBCS_OUTPUT_2_SISO) { - prevLength=cnv->fromUnicodeStatus; - if(prevLength==0) { - /* set the real value */ - prevLength=1; - } - } else { - /* prevent fromUnicodeStatus from being set to something non-0 */ - prevLength=0; - } - - /* sourceIndex=-1 if the current character began in the previous buffer */ - prevSourceIndex=-1; - sourceIndex= c==0 ? 0 : -1; - nextSourceIndex=0; - - /* Get the SI/SO character for the converter */ - siLength = static_cast<uint8_t>(getSISOBytes(SI, cnv->options, siBytes)); - soLength = static_cast<uint8_t>(getSISOBytes(SO, cnv->options, soBytes)); - - /* conversion loop */ - /* - * This is another piece of ugly code: - * A goto into the loop if the converter state contains a first surrogate - * from the previous function call. - * It saves me to check in each loop iteration a check of if(c==0) - * and duplicating the trail-surrogate-handling code in the else - * branch of that check. - * I could not find any other way to get around this other than - * using a function call for the conversion and callback, which would - * be even more inefficient. - * - * Markus Scherer 2000-jul-19 - */ - if(c!=0 && targetCapacity>0) { - goto getTrail; - } - - while(source<sourceLimit) { - /* - * This following test is to see if available input would overflow the output. - * It does not catch output of more than one byte that - * overflows as a result of a multi-byte character or callback output - * from the last source character. - * Therefore, those situations also test for overflows and will - * then break the loop, too. - */ - if(targetCapacity>0) { - /* - * Get a correct Unicode code point: - * a single UChar for a BMP code point or - * a matched surrogate pair for a "supplementary code point". - */ - c=*source++; - ++nextSourceIndex; - if(c<=0x7f && IS_ASCII_ROUNDTRIP(c, asciiRoundtrips)) { - *target++=(uint8_t)c; - if(offsets!=NULL) { - *offsets++=sourceIndex; - prevSourceIndex=sourceIndex; - sourceIndex=nextSourceIndex; - } - --targetCapacity; - c=0; - continue; - } - /* - * utf8Friendly table: Test for <=0xd7ff rather than <=MBCS_FAST_MAX - * to avoid dealing with surrogates. - * MBCS_FAST_MAX must be >=0xd7ff. - */ - if(c<=0xd7ff && mbcsIndex!=NULL) { - value=mbcsIndex[c>>6]; - - /* get the bytes and the length for the output (copied from below and adapted for utf8Friendly data) */ - /* There are only roundtrips (!=0) and no-mapping (==0) entries. */ - switch(outputType) { - case MBCS_OUTPUT_2: - value=((const uint16_t *)bytes)[value +(c&0x3f)]; - if(value<=0xff) { - if(value==0) { - goto unassigned; - } else { - length=1; - } - } else { - length=2; - } - break; - case MBCS_OUTPUT_2_SISO: - /* 1/2-byte stateful with Shift-In/Shift-Out */ - /* - * Save the old state in the converter object - * right here, then change the local prevLength state variable if necessary. - * Then, if this character turns out to be unassigned or a fallback that - * is not taken, the callback code must not save the new state in the converter - * because the new state is for a character that is not output. - * However, the callback must still restore the state from the converter - * in case the callback function changed it for its output. - */ - cnv->fromUnicodeStatus=prevLength; /* save the old state */ - value=((const uint16_t *)bytes)[value +(c&0x3f)]; - if(value<=0xff) { - if(value==0) { - goto unassigned; - } else if(prevLength<=1) { - length=1; - } else { - /* change from double-byte mode to single-byte */ - if (siLength == 1) { - value|=(uint32_t)siBytes[0]<<8; - length = 2; - } else if (siLength == 2) { - value|=(uint32_t)siBytes[1]<<8; - value|=(uint32_t)siBytes[0]<<16; - length = 3; - } - prevLength=1; - } - } else { - if(prevLength==2) { - length=2; - } else { - /* change from single-byte mode to double-byte */ - if (soLength == 1) { - value|=(uint32_t)soBytes[0]<<16; - length = 3; - } else if (soLength == 2) { - value|=(uint32_t)soBytes[1]<<16; - value|=(uint32_t)soBytes[0]<<24; - length = 4; - } - prevLength=2; - } - } - break; - case MBCS_OUTPUT_DBCS_ONLY: - /* table with single-byte results, but only DBCS mappings used */ - value=((const uint16_t *)bytes)[value +(c&0x3f)]; - if(value<=0xff) { - /* no mapping or SBCS result, not taken for DBCS-only */ - goto unassigned; - } else { - length=2; - } - break; - case MBCS_OUTPUT_3: - p=bytes+(value+(c&0x3f))*3; - value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; - if(value<=0xff) { - if(value==0) { - goto unassigned; - } else { - length=1; - } - } else if(value<=0xffff) { - length=2; - } else { - length=3; - } - break; - case MBCS_OUTPUT_4: - value=((const uint32_t *)bytes)[value +(c&0x3f)]; - if(value<=0xff) { - if(value==0) { - goto unassigned; - } else { - length=1; - } - } else if(value<=0xffff) { - length=2; - } else if(value<=0xffffff) { - length=3; - } else { - length=4; - } - break; - case MBCS_OUTPUT_3_EUC: - value=((const uint16_t *)bytes)[value +(c&0x3f)]; - /* EUC 16-bit fixed-length representation */ - if(value<=0xff) { - if(value==0) { - goto unassigned; - } else { - length=1; - } - } else if((value&0x8000)==0) { - value|=0x8e8000; - length=3; - } else if((value&0x80)==0) { - value|=0x8f0080; - length=3; - } else { - length=2; - } - break; - case MBCS_OUTPUT_4_EUC: - p=bytes+(value+(c&0x3f))*3; - value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; - /* EUC 16-bit fixed-length representation applied to the first two bytes */ - if(value<=0xff) { - if(value==0) { - goto unassigned; - } else { - length=1; - } - } else if(value<=0xffff) { - length=2; - } else if((value&0x800000)==0) { - value|=0x8e800000; - length=4; - } else if((value&0x8000)==0) { - value|=0x8f008000; - length=4; - } else { - length=3; - } - break; - default: - /* must not occur */ - /* - * To avoid compiler warnings that value & length may be - * used without having been initialized, we set them here. - * In reality, this is unreachable code. - * Not having a default branch also causes warnings with - * some compilers. - */ - value=0; - length=0; - break; - } - /* output the value */ - } else { - /* - * This also tests if the codepage maps single surrogates. - * If it does, then surrogates are not paired but mapped separately. - * Note that in this case unmatched surrogates are not detected. - */ - if(U16_IS_SURROGATE(c) && !(unicodeMask&UCNV_HAS_SURROGATES)) { - if(U16_IS_SURROGATE_LEAD(c)) { -getTrail: - if(source<sourceLimit) { - /* test the following code unit */ - UChar trail=*source; - if(U16_IS_TRAIL(trail)) { - ++source; - ++nextSourceIndex; - c=U16_GET_SUPPLEMENTARY(c, trail); - if(!(unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { - /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ - cnv->fromUnicodeStatus=prevLength; /* save the old state */ - /* callback(unassigned) */ - goto unassigned; - } - /* convert this supplementary code point */ - /* exit this condition tree */ - } else { - /* this is an unmatched lead code unit (1st surrogate) */ - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - break; - } - } else { - /* no more input */ - break; - } - } else { - /* this is an unmatched trail code unit (2nd surrogate) */ - /* callback(illegal) */ - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - break; - } - } - - /* convert the Unicode code point in c into codepage bytes */ - - /* - * The basic lookup is a triple-stage compact array (trie) lookup. - * For details see the beginning of this file. - * - * Single-byte codepages are handled with a different data structure - * by _MBCSSingle... functions. - * - * The result consists of a 32-bit value from stage 2 and - * a pointer to as many bytes as are stored per character. - * The pointer points to the character's bytes in stage 3. - * Bits 15..0 of the stage 2 entry contain the stage 3 index - * for that pointer, while bits 31..16 are flags for which of - * the 16 characters in the block are roundtrip-assigned. - * - * For 2-byte and 4-byte codepages, the bytes are stored as uint16_t - * respectively as uint32_t, in the platform encoding. - * For 3-byte codepages, the bytes are always stored in big-endian order. - * - * For EUC encodings that use only either 0x8e or 0x8f as the first - * byte of their longest byte sequences, the first two bytes in - * this third stage indicate with their 7th bits whether these bytes - * are to be written directly or actually need to be preceeded by - * one of the two Single-Shift codes. With this, the third stage - * stores one byte fewer per character than the actual maximum length of - * EUC byte sequences. - * - * Other than that, leading zero bytes are removed and the other - * bytes output. A single zero byte may be output if the "assigned" - * bit in stage 2 was on. - * The data structure does not support zero byte output as a fallback, - * and also does not allow output of leading zeros. - */ - stage2Entry=MBCS_STAGE_2_FROM_U(table, c); - - /* get the bytes and the length for the output */ - switch(outputType) { - case MBCS_OUTPUT_2: - value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c); - if(value<=0xff) { - length=1; - } else { - length=2; - } - break; - case MBCS_OUTPUT_2_SISO: - /* 1/2-byte stateful with Shift-In/Shift-Out */ - /* - * Save the old state in the converter object - * right here, then change the local prevLength state variable if necessary. - * Then, if this character turns out to be unassigned or a fallback that - * is not taken, the callback code must not save the new state in the converter - * because the new state is for a character that is not output. - * However, the callback must still restore the state from the converter - * in case the callback function changed it for its output. - */ - cnv->fromUnicodeStatus=prevLength; /* save the old state */ - value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c); - if(value<=0xff) { - if(value==0 && MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c)==0) { - /* no mapping, leave value==0 */ - length=0; - } else if(prevLength<=1) { - length=1; - } else { - /* change from double-byte mode to single-byte */ - if (siLength == 1) { - value|=(uint32_t)siBytes[0]<<8; - length = 2; - } else if (siLength == 2) { - value|=(uint32_t)siBytes[1]<<8; - value|=(uint32_t)siBytes[0]<<16; - length = 3; - } - prevLength=1; - } - } else { - if(prevLength==2) { - length=2; - } else { - /* change from single-byte mode to double-byte */ - if (soLength == 1) { - value|=(uint32_t)soBytes[0]<<16; - length = 3; - } else if (soLength == 2) { - value|=(uint32_t)soBytes[1]<<16; - value|=(uint32_t)soBytes[0]<<24; - length = 4; - } - prevLength=2; - } - } - break; - case MBCS_OUTPUT_DBCS_ONLY: - /* table with single-byte results, but only DBCS mappings used */ - value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c); - if(value<=0xff) { - /* no mapping or SBCS result, not taken for DBCS-only */ - value=stage2Entry=0; /* stage2Entry=0 to reset roundtrip flags */ - length=0; - } else { - length=2; - } - break; - case MBCS_OUTPUT_3: - p=MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c); - value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; - if(value<=0xff) { - length=1; - } else if(value<=0xffff) { - length=2; - } else { - length=3; - } - break; - case MBCS_OUTPUT_4: - value=MBCS_VALUE_4_FROM_STAGE_2(bytes, stage2Entry, c); - if(value<=0xff) { - length=1; - } else if(value<=0xffff) { - length=2; - } else if(value<=0xffffff) { - length=3; - } else { - length=4; - } - break; - case MBCS_OUTPUT_3_EUC: - value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c); - /* EUC 16-bit fixed-length representation */ - if(value<=0xff) { - length=1; - } else if((value&0x8000)==0) { - value|=0x8e8000; - length=3; - } else if((value&0x80)==0) { - value|=0x8f0080; - length=3; - } else { - length=2; - } - break; - case MBCS_OUTPUT_4_EUC: - p=MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c); - value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; - /* EUC 16-bit fixed-length representation applied to the first two bytes */ - if(value<=0xff) { - length=1; - } else if(value<=0xffff) { - length=2; - } else if((value&0x800000)==0) { - value|=0x8e800000; - length=4; - } else if((value&0x8000)==0) { - value|=0x8f008000; - length=4; - } else { - length=3; - } - break; - default: - /* must not occur */ - /* - * To avoid compiler warnings that value & length may be - * used without having been initialized, we set them here. - * In reality, this is unreachable code. - * Not having a default branch also causes warnings with - * some compilers. - */ - value=stage2Entry=0; /* stage2Entry=0 to reset roundtrip flags */ - length=0; - break; - } - - /* is this code point assigned, or do we use fallbacks? */ - if(!(MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c)!=0 || - (UCNV_FROM_U_USE_FALLBACK(cnv, c) && value!=0)) - ) { - /* - * We allow a 0 byte output if the "assigned" bit is set for this entry. - * There is no way with this data structure for fallback output - * to be a zero byte. - */ - -unassigned: - /* try an extension mapping */ - pArgs->source=source; - c=_extFromU(cnv, cnv->sharedData, - c, &source, sourceLimit, - &target, target+targetCapacity, - &offsets, sourceIndex, - pArgs->flush, - pErrorCode); - nextSourceIndex+=(int32_t)(source-pArgs->source); - prevLength=cnv->fromUnicodeStatus; /* restore SISO state */ - - if(U_FAILURE(*pErrorCode)) { - /* not mappable or buffer overflow */ - break; - } else { - /* a mapping was written to the target, continue */ - - /* recalculate the targetCapacity after an extension mapping */ - targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target); - - /* normal end of conversion: prepare for a new character */ - if(offsets!=NULL) { - prevSourceIndex=sourceIndex; - sourceIndex=nextSourceIndex; - } - continue; - } - } - } - - /* write the output character bytes from value and length */ - /* from the first if in the loop we know that targetCapacity>0 */ - if(length<=targetCapacity) { - if(offsets==NULL) { - switch(length) { - /* each branch falls through to the next one */ - case 4: - *target++=(uint8_t)(value>>24); - U_FALLTHROUGH; - case 3: - *target++=(uint8_t)(value>>16); - U_FALLTHROUGH; - case 2: - *target++=(uint8_t)(value>>8); - U_FALLTHROUGH; - case 1: - *target++=(uint8_t)value; - U_FALLTHROUGH; - default: - /* will never occur */ - break; - } - } else { - switch(length) { - /* each branch falls through to the next one */ - case 4: - *target++=(uint8_t)(value>>24); - *offsets++=sourceIndex; - U_FALLTHROUGH; - case 3: - *target++=(uint8_t)(value>>16); - *offsets++=sourceIndex; - U_FALLTHROUGH; - case 2: - *target++=(uint8_t)(value>>8); - *offsets++=sourceIndex; - U_FALLTHROUGH; - case 1: - *target++=(uint8_t)value; - *offsets++=sourceIndex; - U_FALLTHROUGH; - default: - /* will never occur */ - break; - } - } - targetCapacity-=length; - } else { - uint8_t *charErrorBuffer; - - /* - * We actually do this backwards here: - * In order to save an intermediate variable, we output - * first to the overflow buffer what does not fit into the - * regular target. - */ - /* we know that 1<=targetCapacity<length<=4 */ - length-=targetCapacity; - charErrorBuffer=(uint8_t *)cnv->charErrorBuffer; - switch(length) { - /* each branch falls through to the next one */ - case 3: - *charErrorBuffer++=(uint8_t)(value>>16); - U_FALLTHROUGH; - case 2: - *charErrorBuffer++=(uint8_t)(value>>8); - U_FALLTHROUGH; - case 1: - *charErrorBuffer=(uint8_t)value; - U_FALLTHROUGH; - default: - /* will never occur */ - break; - } - cnv->charErrorBufferLength=(int8_t)length; - - /* now output what fits into the regular target */ - value>>=8*length; /* length was reduced by targetCapacity */ - switch(targetCapacity) { - /* each branch falls through to the next one */ - case 3: - *target++=(uint8_t)(value>>16); - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - U_FALLTHROUGH; - case 2: - *target++=(uint8_t)(value>>8); - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - U_FALLTHROUGH; - case 1: - *target++=(uint8_t)value; - if(offsets!=NULL) { - *offsets++=sourceIndex; - } - U_FALLTHROUGH; - default: - /* will never occur */ - break; - } - - /* target overflow */ - targetCapacity=0; - *pErrorCode=U_BUFFER_OVERFLOW_ERROR; - c=0; - break; - } - - /* normal end of conversion: prepare for a new character */ - c=0; - if(offsets!=NULL) { - prevSourceIndex=sourceIndex; - sourceIndex=nextSourceIndex; - } - continue; - } else { - /* target is full */ - *pErrorCode=U_BUFFER_OVERFLOW_ERROR; - break; - } - } - - /* - * the end of the input stream and detection of truncated input - * are handled by the framework, but for EBCDIC_STATEFUL conversion - * we need to emit an SI at the very end - * - * conditions: - * successful - * EBCDIC_STATEFUL in DBCS mode - * end of input and no truncated input - */ - if( U_SUCCESS(*pErrorCode) && - outputType==MBCS_OUTPUT_2_SISO && prevLength==2 && - pArgs->flush && source>=sourceLimit && c==0 - ) { - /* EBCDIC_STATEFUL ending with DBCS: emit an SI to return the output stream to SBCS */ - if(targetCapacity>0) { - *target++=(uint8_t)siBytes[0]; - if (siLength == 2) { - if (targetCapacity<2) { - cnv->charErrorBuffer[0]=(uint8_t)siBytes[1]; - cnv->charErrorBufferLength=1; - *pErrorCode=U_BUFFER_OVERFLOW_ERROR; - } else { - *target++=(uint8_t)siBytes[1]; - } - } - if(offsets!=NULL) { - /* set the last source character's index (sourceIndex points at sourceLimit now) */ - *offsets++=prevSourceIndex; - } - } else { - /* target is full */ - cnv->charErrorBuffer[0]=(uint8_t)siBytes[0]; - if (siLength == 2) { - cnv->charErrorBuffer[1]=(uint8_t)siBytes[1]; - } - cnv->charErrorBufferLength=siLength; - *pErrorCode=U_BUFFER_OVERFLOW_ERROR; - } - prevLength=1; /* we switched into SBCS */ - } - - /* set the converter state back into UConverter */ - cnv->fromUChar32=c; - cnv->fromUnicodeStatus=prevLength; - - /* write back the updated pointers */ - pArgs->source=source; - pArgs->target=(char *)target; - pArgs->offsets=offsets; -} - -/* - * This is another simple conversion function for internal use by other - * conversion implementations. - * It does not use the converter state nor call callbacks. - * It does not handle the EBCDIC swaplfnl option (set in UConverter). - * It handles conversion extensions but not GB 18030. - * - * It converts one single Unicode code point into codepage bytes, encoded - * as one 32-bit value. The function returns the number of bytes in *pValue: - * 1..4 the number of bytes in *pValue - * 0 unassigned (*pValue undefined) - * -1 illegal (currently not used, *pValue undefined) - * - * *pValue will contain the resulting bytes with the last byte in bits 7..0, - * the second to last byte in bits 15..8, etc. - * Currently, the function assumes but does not check that 0<=c<=0x10ffff. - */ -U_CFUNC int32_t -ucnv_MBCSFromUChar32(UConverterSharedData *sharedData, - UChar32 c, uint32_t *pValue, - UBool useFallback) { - const int32_t *cx; - const uint16_t *table; -#if 0 -/* #if 0 because this is not currently used in ICU - reduce code, increase code coverage */ - const uint8_t *p; -#endif - uint32_t stage2Entry; - uint32_t value; - int32_t length; - - /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ - if(c<=0xffff || (sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { - table=sharedData->mbcs.fromUnicodeTable; - - /* convert the Unicode code point in c into codepage bytes (same as in _MBCSFromUnicodeWithOffsets) */ - if(sharedData->mbcs.outputType==MBCS_OUTPUT_1) { - value=MBCS_SINGLE_RESULT_FROM_U(table, (uint16_t *)sharedData->mbcs.fromUnicodeBytes, c); - /* is this code point assigned, or do we use fallbacks? */ - if(useFallback ? value>=0x800 : value>=0xc00) { - *pValue=value&0xff; - return 1; - } - } else /* outputType!=MBCS_OUTPUT_1 */ { - stage2Entry=MBCS_STAGE_2_FROM_U(table, c); - - /* get the bytes and the length for the output */ - switch(sharedData->mbcs.outputType) { - case MBCS_OUTPUT_2: - value=MBCS_VALUE_2_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); - if(value<=0xff) { - length=1; - } else { - length=2; - } - break; -#if 0 -/* #if 0 because this is not currently used in ICU - reduce code, increase code coverage */ - case MBCS_OUTPUT_DBCS_ONLY: - /* table with single-byte results, but only DBCS mappings used */ - value=MBCS_VALUE_2_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); - if(value<=0xff) { - /* no mapping or SBCS result, not taken for DBCS-only */ - value=stage2Entry=0; /* stage2Entry=0 to reset roundtrip flags */ - length=0; - } else { - length=2; - } - break; - case MBCS_OUTPUT_3: - p=MBCS_POINTER_3_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); - value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; - if(value<=0xff) { - length=1; - } else if(value<=0xffff) { - length=2; - } else { - length=3; - } - break; - case MBCS_OUTPUT_4: - value=MBCS_VALUE_4_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); - if(value<=0xff) { - length=1; - } else if(value<=0xffff) { - length=2; - } else if(value<=0xffffff) { - length=3; - } else { - length=4; - } - break; - case MBCS_OUTPUT_3_EUC: - value=MBCS_VALUE_2_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); - /* EUC 16-bit fixed-length representation */ - if(value<=0xff) { - length=1; - } else if((value&0x8000)==0) { - value|=0x8e8000; - length=3; - } else if((value&0x80)==0) { - value|=0x8f0080; - length=3; - } else { - length=2; - } - break; - case MBCS_OUTPUT_4_EUC: - p=MBCS_POINTER_3_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); - value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; - /* EUC 16-bit fixed-length representation applied to the first two bytes */ - if(value<=0xff) { - length=1; - } else if(value<=0xffff) { - length=2; - } else if((value&0x800000)==0) { - value|=0x8e800000; - length=4; - } else if((value&0x8000)==0) { - value|=0x8f008000; - length=4; - } else { - length=3; - } - break; -#endif - default: - /* must not occur */ - return -1; - } - - /* is this code point assigned, or do we use fallbacks? */ - if( MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) || - (FROM_U_USE_FALLBACK(useFallback, c) && value!=0) - ) { - /* - * We allow a 0 byte output if the "assigned" bit is set for this entry. - * There is no way with this data structure for fallback output - * to be a zero byte. - */ - /* assigned */ - *pValue=value; - return length; - } - } - } - - cx=sharedData->mbcs.extIndexes; - if(cx!=NULL) { - length=ucnv_extSimpleMatchFromU(cx, c, pValue, useFallback); - return length>=0 ? length : -length; /* return abs(length); */ - } - - /* unassigned */ - return 0; -} - - -#if 0 -/* - * This function has been moved to ucnv2022.c for inlining. - * This implementation is here only for documentation purposes - */ - -/** - * This version of ucnv_MBCSFromUChar32() is optimized for single-byte codepages. - * It does not handle the EBCDIC swaplfnl option (set in UConverter). - * It does not handle conversion extensions (_extFromU()). - * - * It returns the codepage byte for the code point, or -1 if it is unassigned. - */ -U_CFUNC int32_t -ucnv_MBCSSingleFromUChar32(UConverterSharedData *sharedData, - UChar32 c, - UBool useFallback) { - const uint16_t *table; - int32_t value; - - /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ - if(c>=0x10000 && !(sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { - return -1; - } - - /* convert the Unicode code point in c into codepage bytes (same as in _MBCSFromUnicodeWithOffsets) */ - table=sharedData->mbcs.fromUnicodeTable; - - /* get the byte for the output */ - value=MBCS_SINGLE_RESULT_FROM_U(table, (uint16_t *)sharedData->mbcs.fromUnicodeBytes, c); - /* is this code point assigned, or do we use fallbacks? */ - if(useFallback ? value>=0x800 : value>=0xc00) { - return value&0xff; - } else { - return -1; - } -} -#endif - -/* MBCS-from-UTF-8 conversion functions ------------------------------------- */ - -/* offsets for n-byte UTF-8 sequences that were calculated with ((lead<<6)+trail)<<6+trail... */ -static const UChar32 -utf8_offsets[5]={ 0, 0, 0x3080, 0xE2080, 0x3C82080 }; - -static void U_CALLCONV -ucnv_SBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs, - UConverterToUnicodeArgs *pToUArgs, - UErrorCode *pErrorCode) { - UConverter *utf8, *cnv; - const uint8_t *source, *sourceLimit; - uint8_t *target; - int32_t targetCapacity; - - const uint16_t *table, *sbcsIndex; - const uint16_t *results; - - int8_t oldToULength, toULength, toULimit; - - UChar32 c; - uint8_t b, t1, t2; - - uint32_t asciiRoundtrips; - uint16_t value, minValue = 0; - UBool hasSupplementary; - - /* set up the local pointers */ - utf8=pToUArgs->converter; - cnv=pFromUArgs->converter; - source=(uint8_t *)pToUArgs->source; - sourceLimit=(uint8_t *)pToUArgs->sourceLimit; - target=(uint8_t *)pFromUArgs->target; - targetCapacity=(int32_t)(pFromUArgs->targetLimit-pFromUArgs->target); - - table=cnv->sharedData->mbcs.fromUnicodeTable; - sbcsIndex=cnv->sharedData->mbcs.sbcsIndex; - if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { - results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; - } else { - results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes; - } - asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; - - if(cnv->useFallback) { - /* use all roundtrip and fallback results */ - minValue=0x800; - } else { - /* use only roundtrips and fallbacks from private-use characters */ - minValue=0xc00; - } - hasSupplementary=(UBool)(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY); - - /* get the converter state from the UTF-8 UConverter */ - if(utf8->toULength > 0) { - toULength=oldToULength=utf8->toULength; - toULimit=(int8_t)utf8->mode; - c=(UChar32)utf8->toUnicodeStatus; - } else { - toULength=oldToULength=toULimit=0; - c = 0; - } - - // The conversion loop checks source<sourceLimit only once per 1/2/3-byte character. - // If the buffer ends with a truncated 2- or 3-byte sequence, - // then we reduce the sourceLimit to before that, - // and collect the remaining bytes after the conversion loop. - { - // Do not go back into the bytes that will be read for finishing a partial - // sequence from the previous buffer. - int32_t length=(int32_t)(sourceLimit-source) - (toULimit-oldToULength); - if(length>0) { - uint8_t b1=*(sourceLimit-1); - if(U8_IS_SINGLE(b1)) { - // common ASCII character - } else if(U8_IS_TRAIL(b1) && length>=2) { - uint8_t b2=*(sourceLimit-2); - if(0xe0<=b2 && b2<0xf0 && U8_IS_VALID_LEAD3_AND_T1(b2, b1)) { - // truncated 3-byte sequence - sourceLimit-=2; - } - } else if(0xc2<=b1 && b1<0xf0) { - // truncated 2- or 3-byte sequence - --sourceLimit; - } - } - } - - if(c!=0 && targetCapacity>0) { - utf8->toUnicodeStatus=0; - utf8->toULength=0; - goto moreBytes; - /* - * Note: We could avoid the goto by duplicating some of the moreBytes - * code, but only up to the point of collecting a complete UTF-8 - * sequence; then recurse for the toUBytes[toULength] - * and then continue with normal conversion. - * - * If so, move this code to just after initializing the minimum - * set of local variables for reading the UTF-8 input - * (utf8, source, target, limits but not cnv, table, minValue, etc.). - * - * Potential advantages: - * - avoid the goto - * - oldToULength could become a local variable in just those code blocks - * that deal with buffer boundaries - * - possibly faster if the goto prevents some compiler optimizations - * (this would need measuring to confirm) - * Disadvantage: - * - code duplication - */ - } - - /* conversion loop */ - while(source<sourceLimit) { - if(targetCapacity>0) { - b=*source++; - if(U8_IS_SINGLE(b)) { - /* convert ASCII */ - if(IS_ASCII_ROUNDTRIP(b, asciiRoundtrips)) { - *target++=(uint8_t)b; - --targetCapacity; - continue; - } else { - c=b; - value=SBCS_RESULT_FROM_UTF8(sbcsIndex, results, 0, c); - } - } else { - if(b<0xe0) { - if( /* handle U+0080..U+07FF inline */ - b>=0xc2 && - (t1=(uint8_t)(*source-0x80)) <= 0x3f - ) { - c=b&0x1f; - ++source; - value=SBCS_RESULT_FROM_UTF8(sbcsIndex, results, c, t1); - if(value>=minValue) { - *target++=(uint8_t)value; - --targetCapacity; - continue; - } else { - c=(c<<6)|t1; - } - } else { - c=-1; - } - } else if(b==0xe0) { - if( /* handle U+0800..U+0FFF inline */ - (t1=(uint8_t)(source[0]-0x80)) <= 0x3f && t1 >= 0x20 && - (t2=(uint8_t)(source[1]-0x80)) <= 0x3f - ) { - c=t1; - source+=2; - value=SBCS_RESULT_FROM_UTF8(sbcsIndex, results, c, t2); - if(value>=minValue) { - *target++=(uint8_t)value; - --targetCapacity; - continue; - } else { - c=(c<<6)|t2; - } - } else { - c=-1; - } - } else { - c=-1; - } - - if(c<0) { - /* handle "complicated" and error cases, and continuing partial characters */ - oldToULength=0; - toULength=1; - toULimit=U8_COUNT_BYTES_NON_ASCII(b); - c=b; -moreBytes: - while(toULength<toULimit) { - /* - * The sourceLimit may have been adjusted before the conversion loop - * to stop before a truncated sequence. - * Here we need to use the real limit in case we have two truncated - * sequences at the end. - * See ticket #7492. - */ - if(source<(uint8_t *)pToUArgs->sourceLimit) { - b=*source; - if(icu::UTF8::isValidTrail(c, b, toULength, toULimit)) { - ++source; - ++toULength; - c=(c<<6)+b; - } else { - break; /* sequence too short, stop with toULength<toULimit */ - } - } else { - /* store the partial UTF-8 character, compatible with the regular UTF-8 converter */ - source-=(toULength-oldToULength); - while(oldToULength<toULength) { - utf8->toUBytes[oldToULength++]=*source++; - } - utf8->toUnicodeStatus=c; - utf8->toULength=toULength; - utf8->mode=toULimit; - pToUArgs->source=(char *)source; - pFromUArgs->target=(char *)target; - return; - } - } - - if(toULength==toULimit) { - c-=utf8_offsets[toULength]; - if(toULength<=3) { /* BMP */ - value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); - } else { - /* supplementary code point */ - if(!hasSupplementary) { - /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ - value=0; - } else { - value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); - } - } - } else { - /* error handling: illegal UTF-8 byte sequence */ - source-=(toULength-oldToULength); - while(oldToULength<toULength) { - utf8->toUBytes[oldToULength++]=*source++; - } - utf8->toULength=toULength; - pToUArgs->source=(char *)source; - pFromUArgs->target=(char *)target; - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - return; - } - } - } - - if(value>=minValue) { - /* output the mapping for c */ - *target++=(uint8_t)value; - --targetCapacity; - } else { - /* value<minValue means c is unassigned (unmappable) */ - /* - * Try an extension mapping. - * Pass in no source because we don't have UTF-16 input. - * If we have a partial match on c, we will return and revert - * to UTF-8->UTF-16->charset conversion. - */ - static const UChar nul=0; - const UChar *noSource=&nul; - c=_extFromU(cnv, cnv->sharedData, - c, &noSource, noSource, - &target, target+targetCapacity, - NULL, -1, - pFromUArgs->flush, - pErrorCode); - - if(U_FAILURE(*pErrorCode)) { - /* not mappable or buffer overflow */ - cnv->fromUChar32=c; - break; - } else if(cnv->preFromUFirstCP>=0) { - /* - * Partial match, return and revert to pivoting. - * In normal from-UTF-16 conversion, we would just continue - * but then exit the loop because the extension match would - * have consumed the source. - */ - *pErrorCode=U_USING_DEFAULT_WARNING; - break; - } else { - /* a mapping was written to the target, continue */ - - /* recalculate the targetCapacity after an extension mapping */ - targetCapacity=(int32_t)(pFromUArgs->targetLimit-(char *)target); - } - } - } else { - /* target is full */ - *pErrorCode=U_BUFFER_OVERFLOW_ERROR; - break; - } - } - - /* - * The sourceLimit may have been adjusted before the conversion loop - * to stop before a truncated sequence. - * If so, then collect the truncated sequence now. - */ - if(U_SUCCESS(*pErrorCode) && - cnv->preFromUFirstCP<0 && - source<(sourceLimit=(uint8_t *)pToUArgs->sourceLimit)) { - c=utf8->toUBytes[0]=b=*source++; - toULength=1; - toULimit=U8_COUNT_BYTES(b); - while(source<sourceLimit) { - utf8->toUBytes[toULength++]=b=*source++; - c=(c<<6)+b; - } - utf8->toUnicodeStatus=c; - utf8->toULength=toULength; - utf8->mode=toULimit; - } - - /* write back the updated pointers */ - pToUArgs->source=(char *)source; - pFromUArgs->target=(char *)target; -} - -static void U_CALLCONV -ucnv_DBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs, - UConverterToUnicodeArgs *pToUArgs, - UErrorCode *pErrorCode) { - UConverter *utf8, *cnv; - const uint8_t *source, *sourceLimit; - uint8_t *target; - int32_t targetCapacity; - - const uint16_t *table, *mbcsIndex; - const uint16_t *results; - - int8_t oldToULength, toULength, toULimit; - - UChar32 c; - uint8_t b, t1, t2; - - uint32_t stage2Entry; - uint32_t asciiRoundtrips; - uint16_t value = 0; - UBool hasSupplementary; - - /* set up the local pointers */ - utf8=pToUArgs->converter; - cnv=pFromUArgs->converter; - source=(uint8_t *)pToUArgs->source; - sourceLimit=(uint8_t *)pToUArgs->sourceLimit; - target=(uint8_t *)pFromUArgs->target; - targetCapacity=(int32_t)(pFromUArgs->targetLimit-pFromUArgs->target); - - table=cnv->sharedData->mbcs.fromUnicodeTable; - mbcsIndex=cnv->sharedData->mbcs.mbcsIndex; - if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { - results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; - } else { - results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes; - } - asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; - - hasSupplementary=(UBool)(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY); - - /* get the converter state from the UTF-8 UConverter */ - if(utf8->toULength > 0) { - toULength=oldToULength=utf8->toULength; - toULimit=(int8_t)utf8->mode; - c=(UChar32)utf8->toUnicodeStatus; - } else { - toULength=oldToULength=toULimit=0; - c = 0; - } - - // The conversion loop checks source<sourceLimit only once per 1/2/3-byte character. - // If the buffer ends with a truncated 2- or 3-byte sequence, - // then we reduce the sourceLimit to before that, - // and collect the remaining bytes after the conversion loop. - { - // Do not go back into the bytes that will be read for finishing a partial - // sequence from the previous buffer. - int32_t length=(int32_t)(sourceLimit-source) - (toULimit-oldToULength); - if(length>0) { - uint8_t b1=*(sourceLimit-1); - if(U8_IS_SINGLE(b1)) { - // common ASCII character - } else if(U8_IS_TRAIL(b1) && length>=2) { - uint8_t b2=*(sourceLimit-2); - if(0xe0<=b2 && b2<0xf0 && U8_IS_VALID_LEAD3_AND_T1(b2, b1)) { - // truncated 3-byte sequence - sourceLimit-=2; - } - } else if(0xc2<=b1 && b1<0xf0) { - // truncated 2- or 3-byte sequence - --sourceLimit; - } - } - } - - if(c!=0 && targetCapacity>0) { - utf8->toUnicodeStatus=0; - utf8->toULength=0; - goto moreBytes; - /* See note in ucnv_SBCSFromUTF8() about this goto. */ - } - - /* conversion loop */ - while(source<sourceLimit) { - if(targetCapacity>0) { - b=*source++; - if(U8_IS_SINGLE(b)) { - /* convert ASCII */ - if(IS_ASCII_ROUNDTRIP(b, asciiRoundtrips)) { - *target++=b; - --targetCapacity; - continue; - } else { - value=DBCS_RESULT_FROM_UTF8(mbcsIndex, results, 0, b); - if(value==0) { - c=b; - goto unassigned; - } - } - } else { - if(b>=0xe0) { - if( /* handle U+0800..U+D7FF inline */ - b<=0xed && // do not assume maxFastUChar>0xd7ff - U8_IS_VALID_LEAD3_AND_T1(b, t1=source[0]) && - (t2=(uint8_t)(source[1]-0x80)) <= 0x3f - ) { - c=((b&0xf)<<6)|(t1&0x3f); - source+=2; - value=DBCS_RESULT_FROM_UTF8(mbcsIndex, results, c, t2); - if(value==0) { - c=(c<<6)|t2; - goto unassigned; - } - } else { - c=-1; - } - } else { - if( /* handle U+0080..U+07FF inline */ - b>=0xc2 && - (t1=(uint8_t)(*source-0x80)) <= 0x3f - ) { - c=b&0x1f; - ++source; - value=DBCS_RESULT_FROM_UTF8(mbcsIndex, results, c, t1); - if(value==0) { - c=(c<<6)|t1; - goto unassigned; - } - } else { - c=-1; - } - } - - if(c<0) { - /* handle "complicated" and error cases, and continuing partial characters */ - oldToULength=0; - toULength=1; - toULimit=U8_COUNT_BYTES_NON_ASCII(b); - c=b; -moreBytes: - while(toULength<toULimit) { - /* - * The sourceLimit may have been adjusted before the conversion loop - * to stop before a truncated sequence. - * Here we need to use the real limit in case we have two truncated - * sequences at the end. - * See ticket #7492. - */ - if(source<(uint8_t *)pToUArgs->sourceLimit) { - b=*source; - if(icu::UTF8::isValidTrail(c, b, toULength, toULimit)) { - ++source; - ++toULength; - c=(c<<6)+b; - } else { - break; /* sequence too short, stop with toULength<toULimit */ - } - } else { - /* store the partial UTF-8 character, compatible with the regular UTF-8 converter */ - source-=(toULength-oldToULength); - while(oldToULength<toULength) { - utf8->toUBytes[oldToULength++]=*source++; - } - utf8->toUnicodeStatus=c; - utf8->toULength=toULength; - utf8->mode=toULimit; - pToUArgs->source=(char *)source; - pFromUArgs->target=(char *)target; - return; - } - } - - if(toULength==toULimit) { - c-=utf8_offsets[toULength]; - if(toULength<=3) { /* BMP */ - stage2Entry=MBCS_STAGE_2_FROM_U(table, c); - } else { - /* supplementary code point */ - if(!hasSupplementary) { - /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ - stage2Entry=0; - } else { - stage2Entry=MBCS_STAGE_2_FROM_U(table, c); - } - } - } else { - /* error handling: illegal UTF-8 byte sequence */ - source-=(toULength-oldToULength); - while(oldToULength<toULength) { - utf8->toUBytes[oldToULength++]=*source++; - } - utf8->toULength=toULength; - pToUArgs->source=(char *)source; - pFromUArgs->target=(char *)target; - *pErrorCode=U_ILLEGAL_CHAR_FOUND; - return; - } - - /* get the bytes and the length for the output */ - /* MBCS_OUTPUT_2 */ - value=MBCS_VALUE_2_FROM_STAGE_2(results, stage2Entry, c); - - /* is this code point assigned, or do we use fallbacks? */ - if(!(MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) || - (UCNV_FROM_U_USE_FALLBACK(cnv, c) && value!=0)) - ) { - goto unassigned; - } - } - } - - /* write the output character bytes from value and length */ - /* from the first if in the loop we know that targetCapacity>0 */ - if(value<=0xff) { - /* this is easy because we know that there is enough space */ - *target++=(uint8_t)value; - --targetCapacity; - } else /* length==2 */ { - *target++=(uint8_t)(value>>8); - if(2<=targetCapacity) { - *target++=(uint8_t)value; - targetCapacity-=2; - } else { - cnv->charErrorBuffer[0]=(char)value; - cnv->charErrorBufferLength=1; - - /* target overflow */ - *pErrorCode=U_BUFFER_OVERFLOW_ERROR; - break; - } - } - continue; - -unassigned: - { - /* - * Try an extension mapping. - * Pass in no source because we don't have UTF-16 input. - * If we have a partial match on c, we will return and revert - * to UTF-8->UTF-16->charset conversion. - */ - static const UChar nul=0; - const UChar *noSource=&nul; - c=_extFromU(cnv, cnv->sharedData, - c, &noSource, noSource, - &target, target+targetCapacity, - NULL, -1, - pFromUArgs->flush, - pErrorCode); - - if(U_FAILURE(*pErrorCode)) { - /* not mappable or buffer overflow */ - cnv->fromUChar32=c; - break; - } else if(cnv->preFromUFirstCP>=0) { - /* - * Partial match, return and revert to pivoting. - * In normal from-UTF-16 conversion, we would just continue - * but then exit the loop because the extension match would - * have consumed the source. - */ - *pErrorCode=U_USING_DEFAULT_WARNING; - break; - } else { - /* a mapping was written to the target, continue */ - - /* recalculate the targetCapacity after an extension mapping */ - targetCapacity=(int32_t)(pFromUArgs->targetLimit-(char *)target); - continue; - } - } - } else { - /* target is full */ - *pErrorCode=U_BUFFER_OVERFLOW_ERROR; - break; - } - } - - /* - * The sourceLimit may have been adjusted before the conversion loop - * to stop before a truncated sequence. - * If so, then collect the truncated sequence now. - */ - if(U_SUCCESS(*pErrorCode) && - cnv->preFromUFirstCP<0 && - source<(sourceLimit=(uint8_t *)pToUArgs->sourceLimit)) { - c=utf8->toUBytes[0]=b=*source++; - toULength=1; - toULimit=U8_COUNT_BYTES(b); - while(source<sourceLimit) { - utf8->toUBytes[toULength++]=b=*source++; - c=(c<<6)+b; - } - utf8->toUnicodeStatus=c; - utf8->toULength=toULength; - utf8->mode=toULimit; - } - - /* write back the updated pointers */ - pToUArgs->source=(char *)source; - pFromUArgs->target=(char *)target; -} - -/* miscellaneous ------------------------------------------------------------ */ - -static void U_CALLCONV -ucnv_MBCSGetStarters(const UConverter* cnv, - UBool starters[256], - UErrorCode *) { - const int32_t *state0; - int i; - - state0=cnv->sharedData->mbcs.stateTable[cnv->sharedData->mbcs.dbcsOnlyState]; - for(i=0; i<256; ++i) { - /* all bytes that cause a state transition from state 0 are lead bytes */ - starters[i]= (UBool)MBCS_ENTRY_IS_TRANSITION(state0[i]); - } -} - -/* - * This is an internal function that allows other converter implementations - * to check whether a byte is a lead byte. - */ -U_CFUNC UBool -ucnv_MBCSIsLeadByte(UConverterSharedData *sharedData, char byte) { - return (UBool)MBCS_ENTRY_IS_TRANSITION(sharedData->mbcs.stateTable[0][(uint8_t)byte]); -} - -static void U_CALLCONV -ucnv_MBCSWriteSub(UConverterFromUnicodeArgs *pArgs, - int32_t offsetIndex, - UErrorCode *pErrorCode) { - UConverter *cnv=pArgs->converter; - char *p, *subchar; - char buffer[4]; - int32_t length; - - /* first, select between subChar and subChar1 */ - if( cnv->subChar1!=0 && - (cnv->sharedData->mbcs.extIndexes!=NULL ? - cnv->useSubChar1 : - (cnv->invalidUCharBuffer[0]<=0xff)) - ) { - /* select subChar1 if it is set (not 0) and the unmappable Unicode code point is up to U+00ff (IBM MBCS behavior) */ - subchar=(char *)&cnv->subChar1; - length=1; - } else { - /* select subChar in all other cases */ - subchar=(char *)cnv->subChars; - length=cnv->subCharLen; - } - - /* reset the selector for the next code point */ - cnv->useSubChar1=FALSE; - - if (cnv->sharedData->mbcs.outputType == MBCS_OUTPUT_2_SISO) { - p=buffer; - - /* fromUnicodeStatus contains prevLength */ - switch(length) { - case 1: - if(cnv->fromUnicodeStatus==2) { - /* DBCS mode and SBCS sub char: change to SBCS */ - cnv->fromUnicodeStatus=1; - *p++=UCNV_SI; - } - *p++=subchar[0]; - break; - case 2: - if(cnv->fromUnicodeStatus<=1) { - /* SBCS mode and DBCS sub char: change to DBCS */ - cnv->fromUnicodeStatus=2; - *p++=UCNV_SO; - } - *p++=subchar[0]; - *p++=subchar[1]; - break; - default: - *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; - return; - } - subchar=buffer; - length=(int32_t)(p-buffer); - } - - ucnv_cbFromUWriteBytes(pArgs, subchar, length, offsetIndex, pErrorCode); -} - -U_CFUNC UConverterType -ucnv_MBCSGetType(const UConverter* converter) { - /* SBCS, DBCS, and EBCDIC_STATEFUL are replaced by MBCS, but here we cheat a little */ - if(converter->sharedData->mbcs.countStates==1) { - return (UConverterType)UCNV_SBCS; - } else if((converter->sharedData->mbcs.outputType&0xff)==MBCS_OUTPUT_2_SISO) { - return (UConverterType)UCNV_EBCDIC_STATEFUL; - } else if(converter->sharedData->staticData->minBytesPerChar==2 && converter->sharedData->staticData->maxBytesPerChar==2) { - return (UConverterType)UCNV_DBCS; - } - return (UConverterType)UCNV_MBCS; -} - -#endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */ |