diff options
Diffstat (limited to 'deps/icu-small/source/common/rbbi.cpp')
| -rw-r--r-- | deps/icu-small/source/common/rbbi.cpp | 195 |
1 files changed, 90 insertions, 105 deletions
diff --git a/deps/icu-small/source/common/rbbi.cpp b/deps/icu-small/source/common/rbbi.cpp index 54b289e24d..69f92d94c6 100644 --- a/deps/icu-small/source/common/rbbi.cpp +++ b/deps/icu-small/source/common/rbbi.cpp @@ -64,7 +64,9 @@ UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedBreakIterator) * Constructs a RuleBasedBreakIterator that uses the already-created * tables object that is passed in as a parameter. */ -RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status) { +RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status) + : fSCharIter(UnicodeString()) +{ init(status); fData = new RBBIDataWrapper(data, status); // status checked in constructor if (U_FAILURE(status)) {return;} @@ -80,7 +82,9 @@ RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode // RuleBasedBreakIterator::RuleBasedBreakIterator(const uint8_t *compiledRules, uint32_t ruleLength, - UErrorCode &status) { + UErrorCode &status) + : fSCharIter(UnicodeString()) +{ init(status); if (U_FAILURE(status)) { return; @@ -110,6 +114,7 @@ RuleBasedBreakIterator::RuleBasedBreakIterator(const uint8_t *compiledRules, // //------------------------------------------------------------------------------- RuleBasedBreakIterator::RuleBasedBreakIterator(UDataMemory* udm, UErrorCode &status) + : fSCharIter(UnicodeString()) { init(status); fData = new RBBIDataWrapper(udm, status); // status checked in constructor @@ -130,6 +135,7 @@ RuleBasedBreakIterator::RuleBasedBreakIterator(UDataMemory* udm, UErrorCode &sta RuleBasedBreakIterator::RuleBasedBreakIterator( const UnicodeString &rules, UParseError &parseError, UErrorCode &status) + : fSCharIter(UnicodeString()) { init(status); if (U_FAILURE(status)) {return;} @@ -152,7 +158,9 @@ RuleBasedBreakIterator::RuleBasedBreakIterator( const UnicodeString &rules, // Used when creating a RuleBasedBreakIterator from a set // of rules. //------------------------------------------------------------------------------- -RuleBasedBreakIterator::RuleBasedBreakIterator() { +RuleBasedBreakIterator::RuleBasedBreakIterator() + : fSCharIter(UnicodeString()) +{ UErrorCode status = U_ZERO_ERROR; init(status); } @@ -165,7 +173,8 @@ RuleBasedBreakIterator::RuleBasedBreakIterator() { // //------------------------------------------------------------------------------- RuleBasedBreakIterator::RuleBasedBreakIterator(const RuleBasedBreakIterator& other) -: BreakIterator(other) +: BreakIterator(other), + fSCharIter(UnicodeString()) { UErrorCode status = U_ZERO_ERROR; this->init(status); @@ -177,17 +186,13 @@ RuleBasedBreakIterator::RuleBasedBreakIterator(const RuleBasedBreakIterator& oth * Destructor */ RuleBasedBreakIterator::~RuleBasedBreakIterator() { - if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) { + if (fCharIter != &fSCharIter) { // fCharIter was adopted from the outside. delete fCharIter; } fCharIter = NULL; - delete fSCharIter; - fSCharIter = NULL; - delete fDCharIter; - fDCharIter = NULL; - utext_close(fText); + utext_close(&fText); if (fData != NULL) { fData->removeReference(); @@ -217,26 +222,29 @@ RuleBasedBreakIterator::operator=(const RuleBasedBreakIterator& that) { } BreakIterator::operator=(that); - fBreakType = that.fBreakType; if (fLanguageBreakEngines != NULL) { delete fLanguageBreakEngines; fLanguageBreakEngines = NULL; // Just rebuild for now } // TODO: clone fLanguageBreakEngines from "that" UErrorCode status = U_ZERO_ERROR; - fText = utext_clone(fText, that.fText, FALSE, TRUE, &status); + utext_clone(&fText, &that.fText, FALSE, TRUE, &status); - if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) { + if (fCharIter != &fSCharIter) { delete fCharIter; } - fCharIter = NULL; + fCharIter = &fSCharIter; - if (that.fCharIter != NULL ) { + if (that.fCharIter != NULL && that.fCharIter != &that.fSCharIter) { // This is a little bit tricky - it will intially appear that // this->fCharIter is adopted, even if that->fCharIter was // not adopted. That's ok. fCharIter = that.fCharIter->clone(); } + fSCharIter = that.fSCharIter; + if (fCharIter == NULL) { + fCharIter = &fSCharIter; + } if (fData != NULL) { fData->removeReference(); @@ -269,33 +277,30 @@ RuleBasedBreakIterator::operator=(const RuleBasedBreakIterator& that) { // //----------------------------------------------------------------------------- void RuleBasedBreakIterator::init(UErrorCode &status) { - fText = NULL; fCharIter = NULL; - fSCharIter = NULL; - fDCharIter = NULL; fData = NULL; fPosition = 0; fRuleStatusIndex = 0; fDone = false; fDictionaryCharCount = 0; - fBreakType = UBRK_WORD; // Defaulting BreakType to word gives reasonable - // dictionary behavior for Break Iterators that are - // built from rules. Even better would be the ability to - // declare the type in the rules. - fLanguageBreakEngines = NULL; fUnhandledBreakEngine = NULL; fBreakCache = NULL; fDictionaryCache = NULL; - if (U_FAILURE(status)) { + // Note: IBM xlC is unable to assign or initialize member fText from UTEXT_INITIALIZER. + // fText = UTEXT_INITIALIZER; + static const UText initializedUText = UTEXT_INITIALIZER; + uprv_memcpy(&fText, &initializedUText, sizeof(UText)); + + if (U_FAILURE(status)) { return; } - fText = utext_openUChars(NULL, NULL, 0, &status); + utext_openUChars(&fText, NULL, 0, &status); fDictionaryCache = new DictionaryCache(this, status); fBreakCache = new BreakCache(this, status); - if (U_SUCCESS(status) && (fText == NULL || fDictionaryCache == NULL || fBreakCache == NULL)) { + if (U_SUCCESS(status) && (fDictionaryCache == NULL || fBreakCache == NULL)) { status = U_MEMORY_ALLOCATION_ERROR; } @@ -344,7 +349,7 @@ RuleBasedBreakIterator::operator==(const BreakIterator& that) const { const RuleBasedBreakIterator& that2 = (const RuleBasedBreakIterator&) that; - if (!utext_equals(fText, that2.fText)) { + if (!utext_equals(&fText, &that2.fText)) { // The two break iterators are operating on different text, // or have a different iteration position. // Note that fText's position is always the same as the break iterator's position. @@ -385,7 +390,7 @@ void RuleBasedBreakIterator::setText(UText *ut, UErrorCode &status) { } fBreakCache->reset(); fDictionaryCache->reset(); - fText = utext_clone(fText, ut, FALSE, TRUE, &status); + utext_clone(&fText, ut, FALSE, TRUE, &status); // Set up a dummy CharacterIterator to be returned if anyone // calls getText(). With input from UText, there is no reasonable @@ -393,27 +398,20 @@ void RuleBasedBreakIterator::setText(UText *ut, UErrorCode &status) { // Return one over an empty string instead - this is the closest // we can come to signaling a failure. // (GetText() is obsolete, this failure is sort of OK) - if (fDCharIter == NULL) { - static const UChar c = 0; - fDCharIter = new UCharCharacterIterator(&c, 0); - if (fDCharIter == NULL) { - status = U_MEMORY_ALLOCATION_ERROR; - return; - } - } + fSCharIter.setText(UnicodeString()); - if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) { + if (fCharIter != &fSCharIter) { // existing fCharIter was adopted from the outside. Delete it now. delete fCharIter; } - fCharIter = fDCharIter; + fCharIter = &fSCharIter; this->first(); } UText *RuleBasedBreakIterator::getUText(UText *fillIn, UErrorCode &status) const { - UText *result = utext_clone(fillIn, fText, FALSE, TRUE, &status); + UText *result = utext_clone(fillIn, &fText, FALSE, TRUE, &status); return result; } @@ -439,7 +437,7 @@ void RuleBasedBreakIterator::adoptText(CharacterIterator* newText) { // If we are holding a CharacterIterator adopted from a // previous call to this function, delete it now. - if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) { + if (fCharIter != &fSCharIter) { delete fCharIter; } @@ -450,9 +448,9 @@ RuleBasedBreakIterator::adoptText(CharacterIterator* newText) { if (newText==NULL || newText->startIndex() != 0) { // startIndex !=0 wants to be an error, but there's no way to report it. // Make the iterator text be an empty string. - fText = utext_openUChars(fText, NULL, 0, &status); + utext_openUChars(&fText, NULL, 0, &status); } else { - fText = utext_openCharacterIterator(fText, newText, &status); + utext_openCharacterIterator(&fText, newText, &status); } this->first(); } @@ -467,23 +465,19 @@ RuleBasedBreakIterator::setText(const UnicodeString& newText) { UErrorCode status = U_ZERO_ERROR; fBreakCache->reset(); fDictionaryCache->reset(); - fText = utext_openConstUnicodeString(fText, &newText, &status); + utext_openConstUnicodeString(&fText, &newText, &status); // Set up a character iterator on the string. // Needed in case someone calls getText(). // Can not, unfortunately, do this lazily on the (probably never) // call to getText(), because getText is const. - if (fSCharIter == NULL) { - fSCharIter = new StringCharacterIterator(newText); - } else { - fSCharIter->setText(newText); - } + fSCharIter.setText(newText); - if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) { + if (fCharIter != &fSCharIter) { // old fCharIter was adopted from the outside. Delete it. delete fCharIter; } - fCharIter = fSCharIter; + fCharIter = &fSCharIter; this->first(); } @@ -503,14 +497,14 @@ RuleBasedBreakIterator &RuleBasedBreakIterator::refreshInputText(UText *input, U status = U_ILLEGAL_ARGUMENT_ERROR; return *this; } - int64_t pos = utext_getNativeIndex(fText); + int64_t pos = utext_getNativeIndex(&fText); // Shallow read-only clone of the new UText into the existing input UText - fText = utext_clone(fText, input, FALSE, TRUE, &status); + utext_clone(&fText, input, FALSE, TRUE, &status); if (U_FAILURE(status)) { return *this; } - utext_setNativeIndex(fText, pos); - if (utext_getNativeIndex(fText) != pos) { + utext_setNativeIndex(&fText, pos); + if (utext_getNativeIndex(&fText) != pos) { // Sanity check. The new input utext is supposed to have the exact same // contents as the old. If we can't set to the same position, it doesn't. // The contents underlying the old utext might be invalid at this point, @@ -540,7 +534,7 @@ int32_t RuleBasedBreakIterator::first(void) { * @return The text's past-the-end offset. */ int32_t RuleBasedBreakIterator::last(void) { - int32_t endPos = (int32_t)utext_nativeLength(fText); + int32_t endPos = (int32_t)utext_nativeLength(&fText); UBool endShouldBeBoundary = isBoundary(endPos); // Has side effect of setting iterator position. (void)endShouldBeBoundary; U_ASSERT(endShouldBeBoundary); @@ -611,8 +605,8 @@ int32_t RuleBasedBreakIterator::following(int32_t startPos) { // Move requested offset to a code point start. It might be on a trail surrogate, // or on a trail byte if the input is UTF-8. Or it may be beyond the end of the text. - utext_setNativeIndex(fText, startPos); - startPos = (int32_t)utext_getNativeIndex(fText); + utext_setNativeIndex(&fText, startPos); + startPos = (int32_t)utext_getNativeIndex(&fText); UErrorCode status = U_ZERO_ERROR; fBreakCache->following(startPos, status); @@ -626,15 +620,15 @@ int32_t RuleBasedBreakIterator::following(int32_t startPos) { * @return The position of the last boundary before the starting position. */ int32_t RuleBasedBreakIterator::preceding(int32_t offset) { - if (fText == NULL || offset > utext_nativeLength(fText)) { + if (offset > utext_nativeLength(&fText)) { return last(); } // Move requested offset to a code point start. It might be on a trail surrogate, // or on a trail byte if the input is UTF-8. - utext_setNativeIndex(fText, offset); - int32_t adjustedOffset = utext_getNativeIndex(fText); + utext_setNativeIndex(&fText, offset); + int32_t adjustedOffset = utext_getNativeIndex(&fText); UErrorCode status = U_ZERO_ERROR; fBreakCache->preceding(adjustedOffset, status); @@ -660,8 +654,8 @@ UBool RuleBasedBreakIterator::isBoundary(int32_t offset) { // Note that isBoundary() is always be false for offsets that are not on code point boundaries. // But we still need the side effect of leaving iteration at the following boundary. - utext_setNativeIndex(fText, offset); - int32_t adjustedOffset = utext_getNativeIndex(fText); + utext_setNativeIndex(&fText, offset); + int32_t adjustedOffset = utext_getNativeIndex(&fText); bool result = false; UErrorCode status = U_ZERO_ERROR; @@ -669,7 +663,7 @@ UBool RuleBasedBreakIterator::isBoundary(int32_t offset) { result = (fBreakCache->current() == offset); } - if (result && adjustedOffset < offset && utext_char32At(fText, offset) == U_SENTINEL) { + if (result && adjustedOffset < offset && utext_char32At(&fText, offset) == U_SENTINEL) { // Original offset is beyond the end of the text. Return FALSE, it's not a boundary, // but the iteration position remains set to the end of the text, which is a boundary. return FALSE; @@ -789,9 +783,9 @@ int32_t RuleBasedBreakIterator::handleNext() { // if we're already at the end of the text, return DONE. initialPosition = fPosition; - UTEXT_SETNATIVEINDEX(fText, initialPosition); + UTEXT_SETNATIVEINDEX(&fText, initialPosition); result = initialPosition; - c = UTEXT_NEXT32(fText); + c = UTEXT_NEXT32(&fText); if (c==U_SENTINEL) { fDone = TRUE; return UBRK_DONE; @@ -854,7 +848,7 @@ int32_t RuleBasedBreakIterator::handleNext() { #ifdef RBBI_DEBUG if (gTrace) { - RBBIDebugPrintf(" %4ld ", utext_getNativeIndex(fText)); + RBBIDebugPrintf(" %4ld ", utext_getNativeIndex(&fText)); if (0x20<=c && c<0x7f) { RBBIDebugPrintf("\"%c\" ", c); } else { @@ -867,9 +861,7 @@ int32_t RuleBasedBreakIterator::handleNext() { // State Transition - move machine to its next state // - // Note: fNextState is defined as uint16_t[2], but we are casting - // a generated RBBI table to RBBIStateTableRow and some tables - // actually have more than 2 categories. + // fNextState is a variable-length array. U_ASSERT(category<fData->fHeader->fCatCount); state = row->fNextState[category]; /*Not accessing beyond memory*/ row = (RBBIStateTableRow *) @@ -880,7 +872,7 @@ int32_t RuleBasedBreakIterator::handleNext() { if (row->fAccepting == -1) { // Match found, common case. if (mode != RBBI_START) { - result = (int32_t)UTEXT_GETNATIVEINDEX(fText); + result = (int32_t)UTEXT_GETNATIVEINDEX(&fText); } fRuleStatusIndex = row->fTagIdx; // Remember the break status (tag) values. } @@ -898,7 +890,7 @@ int32_t RuleBasedBreakIterator::handleNext() { int16_t rule = row->fLookAhead; if (rule != 0) { // At the position of a '/' in a look-ahead match. Record it. - int32_t pos = (int32_t)UTEXT_GETNATIVEINDEX(fText); + int32_t pos = (int32_t)UTEXT_GETNATIVEINDEX(&fText); lookAheadMatches.setPosition(rule, pos); } @@ -914,7 +906,7 @@ int32_t RuleBasedBreakIterator::handleNext() { // the input position. The next iteration will be processing the // first real input character. if (mode == RBBI_RUN) { - c = UTEXT_NEXT32(fText); + c = UTEXT_NEXT32(&fText); } else { if (mode == RBBI_START) { mode = RBBI_RUN; @@ -928,9 +920,9 @@ int32_t RuleBasedBreakIterator::handleNext() { // (This really indicates a defect in the break rules. They should always match // at least one character.) if (result == initialPosition) { - utext_setNativeIndex(fText, initialPosition); - utext_next32(fText); - result = (int32_t)utext_getNativeIndex(fText); + utext_setNativeIndex(&fText, initialPosition); + utext_next32(&fText); + result = (int32_t)utext_getNativeIndex(&fText); fRuleStatusIndex = 0; } @@ -965,7 +957,7 @@ int32_t RuleBasedBreakIterator::handlePrevious(int32_t fromPosition) { int32_t initialPosition = 0; const RBBIStateTable *stateTable = fData->fSafeRevTable; - UTEXT_SETNATIVEINDEX(fText, fromPosition); + UTEXT_SETNATIVEINDEX(&fText, fromPosition); #ifdef RBBI_DEBUG if (gTrace) { RBBIDebugPuts("Handle Previous pos char state category"); @@ -973,14 +965,14 @@ int32_t RuleBasedBreakIterator::handlePrevious(int32_t fromPosition) { #endif // if we're already at the start of the text, return DONE. - if (fText == NULL || fData == NULL || UTEXT_GETNATIVEINDEX(fText)==0) { + if (fData == NULL || UTEXT_GETNATIVEINDEX(&fText)==0) { return BreakIterator::DONE; } // Set up the starting char. - initialPosition = (int32_t)UTEXT_GETNATIVEINDEX(fText); + initialPosition = (int32_t)UTEXT_GETNATIVEINDEX(&fText); result = initialPosition; - c = UTEXT_PREVIOUS32(fText); + c = UTEXT_PREVIOUS32(&fText); // Set the initial state for the state machine state = START_STATE; @@ -1028,7 +1020,7 @@ int32_t RuleBasedBreakIterator::handlePrevious(int32_t fromPosition) { #ifdef RBBI_DEBUG if (gTrace) { - RBBIDebugPrintf(" %4d ", (int32_t)utext_getNativeIndex(fText)); + RBBIDebugPrintf(" %4d ", (int32_t)utext_getNativeIndex(&fText)); if (0x20<=c && c<0x7f) { RBBIDebugPrintf("\"%c\" ", c); } else { @@ -1041,9 +1033,7 @@ int32_t RuleBasedBreakIterator::handlePrevious(int32_t fromPosition) { // State Transition - move machine to its next state // - // Note: fNextState is defined as uint16_t[2], but we are casting - // a generated RBBI table to RBBIStateTableRow and some tables - // actually have more than 2 categories. + // fNextState is a variable-length array. U_ASSERT(category<fData->fHeader->fCatCount); state = row->fNextState[category]; /*Not accessing beyond memory*/ row = (RBBIStateTableRow *) @@ -1051,7 +1041,7 @@ int32_t RuleBasedBreakIterator::handlePrevious(int32_t fromPosition) { if (row->fAccepting == -1) { // Match found, common case. - result = (int32_t)UTEXT_GETNATIVEINDEX(fText); + result = (int32_t)UTEXT_GETNATIVEINDEX(&fText); } int16_t completedRule = row->fAccepting; @@ -1059,14 +1049,14 @@ int32_t RuleBasedBreakIterator::handlePrevious(int32_t fromPosition) { // Lookahead match is completed. int32_t lookaheadResult = lookAheadMatches.getPosition(completedRule); if (lookaheadResult >= 0) { - UTEXT_SETNATIVEINDEX(fText, lookaheadResult); + UTEXT_SETNATIVEINDEX(&fText, lookaheadResult); return lookaheadResult; } } int16_t rule = row->fLookAhead; if (rule != 0) { // At the position of a '/' in a look-ahead match. Record it. - int32_t pos = (int32_t)UTEXT_GETNATIVEINDEX(fText); + int32_t pos = (int32_t)UTEXT_GETNATIVEINDEX(&fText); lookAheadMatches.setPosition(rule, pos); } @@ -1082,7 +1072,7 @@ int32_t RuleBasedBreakIterator::handlePrevious(int32_t fromPosition) { // the input position. The next iteration will be processing the // first real input character. if (mode == RBBI_RUN) { - c = UTEXT_PREVIOUS32(fText); + c = UTEXT_PREVIOUS32(&fText); } else { if (mode == RBBI_START) { mode = RBBI_RUN; @@ -1096,9 +1086,9 @@ int32_t RuleBasedBreakIterator::handlePrevious(int32_t fromPosition) { // (This really indicates a defect in the break rules. They should always match // at least one character.) if (result == initialPosition) { - UTEXT_SETNATIVEINDEX(fText, initialPosition); - UTEXT_PREVIOUS32(fText); - result = (int32_t)UTEXT_GETNATIVEINDEX(fText); + UTEXT_SETNATIVEINDEX(&fText, initialPosition); + UTEXT_PREVIOUS32(&fText); + result = (int32_t)UTEXT_GETNATIVEINDEX(&fText); } #ifdef RBBI_DEBUG @@ -1247,7 +1237,7 @@ static void U_CALLCONV initLanguageFactories() { static const LanguageBreakEngine* -getLanguageBreakEngineFromFactory(UChar32 c, int32_t breakType) +getLanguageBreakEngineFromFactory(UChar32 c) { umtx_initOnce(gLanguageBreakFactoriesInitOnce, &initLanguageFactories); if (gLanguageBreakFactories == NULL) { @@ -1258,7 +1248,7 @@ getLanguageBreakEngineFromFactory(UChar32 c, int32_t breakType) const LanguageBreakEngine *lbe = NULL; while (--i >= 0) { LanguageBreakFactory *factory = (LanguageBreakFactory *)(gLanguageBreakFactories->elementAt(i)); - lbe = factory->getEngineFor(c, breakType); + lbe = factory->getEngineFor(c); if (lbe != NULL) { break; } @@ -1290,14 +1280,14 @@ RuleBasedBreakIterator::getLanguageBreakEngine(UChar32 c) { int32_t i = fLanguageBreakEngines->size(); while (--i >= 0) { lbe = (const LanguageBreakEngine *)(fLanguageBreakEngines->elementAt(i)); - if (lbe->handles(c, fBreakType)) { + if (lbe->handles(c)) { return lbe; } } // No existing dictionary took the character. See if a factory wants to // give us a new LanguageBreakEngine for this character. - lbe = getLanguageBreakEngineFromFactory(c, fBreakType); + lbe = getLanguageBreakEngineFromFactory(c); // If we got one, use it and push it on our stack. if (lbe != NULL) { @@ -1313,6 +1303,7 @@ RuleBasedBreakIterator::getLanguageBreakEngine(UChar32 c) { fUnhandledBreakEngine = new UnhandledEngine(status); if (U_SUCCESS(status) && fUnhandledBreakEngine == NULL) { status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; } // Put it last so that scripts for which we have an engine get tried // first. @@ -1327,25 +1318,19 @@ RuleBasedBreakIterator::getLanguageBreakEngine(UChar32 c) { // Tell the reject engine about the character; at its discretion, it may // add more than just the one character. - fUnhandledBreakEngine->handleCharacter(c, fBreakType); + fUnhandledBreakEngine->handleCharacter(c); return fUnhandledBreakEngine; } - - -/*int32_t RuleBasedBreakIterator::getBreakType() const { - return fBreakType; -}*/ - -void RuleBasedBreakIterator::setBreakType(int32_t type) { - fBreakType = type; -} - void RuleBasedBreakIterator::dumpCache() { fBreakCache->dumpCache(); } +void RuleBasedBreakIterator::dumpTables() { + fData->printData(); +} + /** * Returns the description used to create this iterator */ |