diff options
author | Steven R. Loomis <srloomis@us.ibm.com> | 2018-10-17 09:43:52 -0700 |
---|---|---|
committer | Steven R. Loomis <srloomis@us.ibm.com> | 2018-10-24 08:27:36 -0700 |
commit | 6786ff4d3688512d8b717ec24188818ac5493d0b (patch) | |
tree | ab18b7a66afee52420fe0bedf7b38eb93f671373 /deps/icu-small/source/i18n/unicode/translit.h | |
parent | d8f2d2726143ecfbf99b90b7bbbc6f41b3cd95fc (diff) | |
download | android-node-v8-6786ff4d3688512d8b717ec24188818ac5493d0b.tar.gz android-node-v8-6786ff4d3688512d8b717ec24188818ac5493d0b.tar.bz2 android-node-v8-6786ff4d3688512d8b717ec24188818ac5493d0b.zip |
deps: icu 63.1 bump (CLDR 34)
- Full release notes: http://site.icu-project.org/download/63
Fixes: https://github.com/nodejs/node/issues/22344
PR-URL: https://github.com/nodejs/node/pull/23715
Reviewed-By: Refael Ackermann <refack@gmail.com>
Reviewed-By: Richard Lau <riclau@uk.ibm.com>
Reviewed-By: Gus Caplan <me@gus.host>
Reviewed-By: Michael Dawson <michael_dawson@ca.ibm.com>
Diffstat (limited to 'deps/icu-small/source/i18n/unicode/translit.h')
-rw-r--r-- | deps/icu-small/source/i18n/unicode/translit.h | 291 |
1 files changed, 269 insertions, 22 deletions
diff --git a/deps/icu-small/source/i18n/unicode/translit.h b/deps/icu-small/source/i18n/unicode/translit.h index dc31d97bc6..6b4888145f 100644 --- a/deps/icu-small/source/i18n/unicode/translit.h +++ b/deps/icu-small/source/i18n/unicode/translit.h @@ -31,7 +31,6 @@ U_NAMESPACE_BEGIN class UnicodeFilter; class UnicodeSet; -class CompoundTransliterator; class TransliteratorParser; class NormalizationTransliterator; class TransliteratorIDParser; @@ -77,8 +76,7 @@ class TransliteratorIDParser; * transliteration. For example, given a string <code>input</code> * and a transliterator <code>t</code>, the call * - * \htmlonly<blockquote>\endhtmlonly<code>String result = t.transliterate(input); - * </code>\htmlonly</blockquote>\endhtmlonly + * String result = t.transliterate(input); * * will transliterate it and return the result. Other methods allow * the client to specify a substring to be transliterated and to use @@ -98,22 +96,20 @@ class TransliteratorIDParser; * contents of the buffer may show text being modified as each new * character arrives. * - * <p>Consider the simple <code>RuleBasedTransliterator</code>: - * - * \htmlonly<blockquote>\endhtmlonly<code> - * th>{theta}<br> - * t>{tau} - * </code>\htmlonly</blockquote>\endhtmlonly + * <p>Consider the simple rule-based Transliterator: + * <pre> + * th>{theta} + * t>{tau} + * </pre> * * When the user types 't', nothing will happen, since the * transliterator is waiting to see if the next character is 'h'. To * remedy this, we introduce the notion of a cursor, marked by a '|' * in the output string: - * - * \htmlonly<blockquote>\endhtmlonly<code> - * t>|{tau}<br> - * {tau}h>{theta} - * </code>\htmlonly</blockquote>\endhtmlonly + * <pre> + * t>|{tau} + * {tau}h>{theta} + * </pre> * * Now when the user types 't', tau appears, and if the next character * is 'h', the tau changes to a theta. This is accomplished by @@ -135,7 +131,7 @@ class TransliteratorIDParser; * which the transliterator last stopped, either because it reached * the end, or because it required more characters to disambiguate * between possible inputs. The <code>CURSOR</code> can also be - * explicitly set by rules in a <code>RuleBasedTransliterator</code>. + * explicitly set by rules in a rule-based Transliterator. * Any characters before the <code>CURSOR</code> index are frozen; * future keyboard transliteration calls within this input sequence * will not change them. New text is inserted at the @@ -237,6 +233,255 @@ class TransliteratorIDParser; * if the performance of these methods can be improved over the * performance obtained by the default implementations in this class. * + * <p><b>Rule syntax</b> + * + * <p>A set of rules determines how to perform translations. + * Rules within a rule set are separated by semicolons (';'). + * To include a literal semicolon, prefix it with a backslash ('\'). + * Unicode Pattern_White_Space is ignored. + * If the first non-blank character on a line is '#', + * the entire line is ignored as a comment. + * + * <p>Each set of rules consists of two groups, one forward, and one + * reverse. This is a convention that is not enforced; rules for one + * direction may be omitted, with the result that translations in + * that direction will not modify the source text. In addition, + * bidirectional forward-reverse rules may be specified for + * symmetrical transformations. + * + * <p>Note: Another description of the Transliterator rule syntax is available in + * <a href="https://www.unicode.org/reports/tr35/tr35-general.html#Transform_Rules_Syntax">section + * Transform Rules Syntax of UTS #35: Unicode LDML</a>. + * The rules are shown there using arrow symbols ← and → and ↔. + * ICU supports both those and the equivalent ASCII symbols < and > and <>. + * + * <p>Rule statements take one of the following forms: + * + * <dl> + * <dt><code>$alefmadda=\\u0622;</code></dt> + * <dd><strong>Variable definition.</strong> The name on the + * left is assigned the text on the right. In this example, + * after this statement, instances of the left hand name, + * "<code>$alefmadda</code>", will be replaced by + * the Unicode character U+0622. Variable names must begin + * with a letter and consist only of letters, digits, and + * underscores. Case is significant. Duplicate names cause + * an exception to be thrown, that is, variables cannot be + * redefined. The right hand side may contain well-formed + * text of any length, including no text at all ("<code>$empty=;</code>"). + * The right hand side may contain embedded <code>UnicodeSet</code> + * patterns, for example, "<code>$softvowel=[eiyEIY]</code>".</dd> + * <dt><code>ai>$alefmadda;</code></dt> + * <dd><strong>Forward translation rule.</strong> This rule + * states that the string on the left will be changed to the + * string on the right when performing forward + * transliteration.</dd> + * <dt><code>ai<$alefmadda;</code></dt> + * <dd><strong>Reverse translation rule.</strong> This rule + * states that the string on the right will be changed to + * the string on the left when performing reverse + * transliteration.</dd> + * </dl> + * + * <dl> + * <dt><code>ai<>$alefmadda;</code></dt> + * <dd><strong>Bidirectional translation rule.</strong> This + * rule states that the string on the right will be changed + * to the string on the left when performing forward + * transliteration, and vice versa when performing reverse + * transliteration.</dd> + * </dl> + * + * <p>Translation rules consist of a <em>match pattern</em> and an <em>output + * string</em>. The match pattern consists of literal characters, + * optionally preceded by context, and optionally followed by + * context. Context characters, like literal pattern characters, + * must be matched in the text being transliterated. However, unlike + * literal pattern characters, they are not replaced by the output + * text. For example, the pattern "<code>abc{def}</code>" + * indicates the characters "<code>def</code>" must be + * preceded by "<code>abc</code>" for a successful match. + * If there is a successful match, "<code>def</code>" will + * be replaced, but not "<code>abc</code>". The final '<code>}</code>' + * is optional, so "<code>abc{def</code>" is equivalent to + * "<code>abc{def}</code>". Another example is "<code>{123}456</code>" + * (or "<code>123}456</code>") in which the literal + * pattern "<code>123</code>" must be followed by "<code>456</code>". + * + * <p>The output string of a forward or reverse rule consists of + * characters to replace the literal pattern characters. If the + * output string contains the character '<code>|</code>', this is + * taken to indicate the location of the <em>cursor</em> after + * replacement. The cursor is the point in the text at which the + * next replacement, if any, will be applied. The cursor is usually + * placed within the replacement text; however, it can actually be + * placed into the precending or following context by using the + * special character '@'. Examples: + * + * <pre> + * a {foo} z > | @ bar; # foo -> bar, move cursor before a + * {foo} xyz > bar @@|; # foo -> bar, cursor between y and z + * </pre> + * + * <p><b>UnicodeSet</b> + * + * <p><code>UnicodeSet</code> patterns may appear anywhere that + * makes sense. They may appear in variable definitions. + * Contrariwise, <code>UnicodeSet</code> patterns may themselves + * contain variable references, such as "<code>$a=[a-z];$not_a=[^$a]</code>", + * or "<code>$range=a-z;$ll=[$range]</code>". + * + * <p><code>UnicodeSet</code> patterns may also be embedded directly + * into rule strings. Thus, the following two rules are equivalent: + * + * <pre> + * $vowel=[aeiou]; $vowel>'*'; # One way to do this + * [aeiou]>'*'; # Another way + * </pre> + * + * <p>See {@link UnicodeSet} for more documentation and examples. + * + * <p><b>Segments</b> + * + * <p>Segments of the input string can be matched and copied to the + * output string. This makes certain sets of rules simpler and more + * general, and makes reordering possible. For example: + * + * <pre> + * ([a-z]) > $1 $1; # double lowercase letters + * ([:Lu:]) ([:Ll:]) > $2 $1; # reverse order of Lu-Ll pairs + * </pre> + * + * <p>The segment of the input string to be copied is delimited by + * "<code>(</code>" and "<code>)</code>". Up to + * nine segments may be defined. Segments may not overlap. In the + * output string, "<code>$1</code>" through "<code>$9</code>" + * represent the input string segments, in left-to-right order of + * definition. + * + * <p><b>Anchors</b> + * + * <p>Patterns can be anchored to the beginning or the end of the text. This is done with the + * special characters '<code>^</code>' and '<code>$</code>'. For example: + * + * <pre> + * ^ a > 'BEG_A'; # match 'a' at start of text + * a > 'A'; # match other instances of 'a' + * z $ > 'END_Z'; # match 'z' at end of text + * z > 'Z'; # match other instances of 'z' + * </pre> + * + * <p>It is also possible to match the beginning or the end of the text using a <code>UnicodeSet</code>. + * This is done by including a virtual anchor character '<code>$</code>' at the end of the + * set pattern. Although this is usually the match chafacter for the end anchor, the set will + * match either the beginning or the end of the text, depending on its placement. For + * example: + * + * <pre> + * $x = [a-z$]; # match 'a' through 'z' OR anchor + * $x 1 > 2; # match '1' after a-z or at the start + * 3 $x > 4; # match '3' before a-z or at the end + * </pre> + * + * <p><b>Example</b> + * + * <p>The following example rules illustrate many of the features of + * the rule language. + * + * <table border="0" cellpadding="4"> + * <tr> + * <td style="vertical-align: top;">Rule 1.</td> + * <td style="vertical-align: top; write-space: nowrap;"><code>abc{def}>x|y</code></td> + * </tr> + * <tr> + * <td style="vertical-align: top;">Rule 2.</td> + * <td style="vertical-align: top; write-space: nowrap;"><code>xyz>r</code></td> + * </tr> + * <tr> + * <td style="vertical-align: top;">Rule 3.</td> + * <td style="vertical-align: top; write-space: nowrap;"><code>yz>q</code></td> + * </tr> + * </table> + * + * <p>Applying these rules to the string "<code>adefabcdefz</code>" + * yields the following results: + * + * <table border="0" cellpadding="4"> + * <tr> + * <td style="vertical-align: top; write-space: nowrap;"><code>|adefabcdefz</code></td> + * <td style="vertical-align: top;">Initial state, no rules match. Advance + * cursor.</td> + * </tr> + * <tr> + * <td style="vertical-align: top; write-space: nowrap;"><code>a|defabcdefz</code></td> + * <td style="vertical-align: top;">Still no match. Rule 1 does not match + * because the preceding context is not present.</td> + * </tr> + * <tr> + * <td style="vertical-align: top; write-space: nowrap;"><code>ad|efabcdefz</code></td> + * <td style="vertical-align: top;">Still no match. Keep advancing until + * there is a match...</td> + * </tr> + * <tr> + * <td style="vertical-align: top; write-space: nowrap;"><code>ade|fabcdefz</code></td> + * <td style="vertical-align: top;">...</td> + * </tr> + * <tr> + * <td style="vertical-align: top; write-space: nowrap;"><code>adef|abcdefz</code></td> + * <td style="vertical-align: top;">...</td> + * </tr> + * <tr> + * <td style="vertical-align: top; write-space: nowrap;"><code>adefa|bcdefz</code></td> + * <td style="vertical-align: top;">...</td> + * </tr> + * <tr> + * <td style="vertical-align: top; write-space: nowrap;"><code>adefab|cdefz</code></td> + * <td style="vertical-align: top;">...</td> + * </tr> + * <tr> + * <td style="vertical-align: top; write-space: nowrap;"><code>adefabc|defz</code></td> + * <td style="vertical-align: top;">Rule 1 matches; replace "<code>def</code>" + * with "<code>xy</code>" and back up the cursor + * to before the '<code>y</code>'.</td> + * </tr> + * <tr> + * <td style="vertical-align: top; write-space: nowrap;"><code>adefabcx|yz</code></td> + * <td style="vertical-align: top;">Although "<code>xyz</code>" is + * present, rule 2 does not match because the cursor is + * before the '<code>y</code>', not before the '<code>x</code>'. + * Rule 3 does match. Replace "<code>yz</code>" + * with "<code>q</code>".</td> + * </tr> + * <tr> + * <td style="vertical-align: top; write-space: nowrap;"><code>adefabcxq|</code></td> + * <td style="vertical-align: top;">The cursor is at the end; + * transliteration is complete.</td> + * </tr> + * </table> + * + * <p>The order of rules is significant. If multiple rules may match + * at some point, the first matching rule is applied. + * + * <p>Forward and reverse rules may have an empty output string. + * Otherwise, an empty left or right hand side of any statement is a + * syntax error. + * + * <p>Single quotes are used to quote any character other than a + * digit or letter. To specify a single quote itself, inside or + * outside of quotes, use two single quotes in a row. For example, + * the rule "<code>'>'>o''clock</code>" changes the + * string "<code>></code>" to the string "<code>o'clock</code>". + * + * <p><b>Notes</b> + * + * <p>While a Transliterator is being built from rules, it checks that + * the rules are added in proper order. For example, if the rule + * "a>x" is followed by the rule "ab>y", + * then the second rule will throw an exception. The reason is that + * the second rule can never be triggered, since the first rule + * always matches anything it matches. In other words, the first + * rule <em>masks</em> the second rule. + * * @author Alan Liu * @stable ICU 2.0 */ @@ -499,9 +744,9 @@ public: * for details. * @param text the buffer holding transliterated and * untransliterated text - * @param index an array of three integers. See {@link #transliterate(Replaceable&, UTransPosition&, const UnicodeString*, UErrorCode&) const }. + * @param index an array of three integers. * @param status Output param to filled in with a success or an error. - * @see #transliterate(Replaceable, int[], String) + * @see #transliterate(Replaceable&, UTransPosition&, const UnicodeString&, UErrorCode &) const * @stable ICU 2.0 */ virtual void transliterate(Replaceable& text, UTransPosition& index, @@ -632,7 +877,7 @@ public: /** * Transliterate a substring of text, as specified by index, taking filters * into account. This method is for subclasses that need to delegate to - * another transliterator, such as CompoundTransliterator. + * another transliterator. * @param text the text to be transliterated * @param index the position indices * @param incremental if TRUE, then assume more characters may be inserted @@ -846,17 +1091,19 @@ public: /** * Returns a <code>Transliterator</code> object constructed from - * the given rule string. This will be a RuleBasedTransliterator, + * the given rule string. This will be a rule-based Transliterator, * if the rule string contains only rules, or a - * CompoundTransliterator, if it contains ID blocks, or a - * NullTransliterator, if it contains ID blocks which parse as + * compound Transliterator, if it contains ID blocks, or a + * null Transliterator, if it contains ID blocks which parse as * empty for the given direction. + * * @param ID the id for the transliterator. * @param rules rules, separated by ';' * @param dir either FORWARD or REVERSE. - * @param parseError Struct to recieve information on position + * @param parseError Struct to receive information on position * of error if an error is encountered * @param status Output param set to success/failure code. + * @return a newly created Transliterator * @stable ICU 2.0 */ static Transliterator* U_EXPORT2 createFromRules(const UnicodeString& ID, |