summaryrefslogtreecommitdiff
path: root/deps/node/deps/icu-small/source/i18n/csrucode.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'deps/node/deps/icu-small/source/i18n/csrucode.cpp')
-rw-r--r--deps/node/deps/icu-small/source/i18n/csrucode.cpp199
1 files changed, 0 insertions, 199 deletions
diff --git a/deps/node/deps/icu-small/source/i18n/csrucode.cpp b/deps/node/deps/icu-small/source/i18n/csrucode.cpp
deleted file mode 100644
index b84011c2..00000000
--- a/deps/node/deps/icu-small/source/i18n/csrucode.cpp
+++ /dev/null
@@ -1,199 +0,0 @@
-// © 2016 and later: Unicode, Inc. and others.
-// License & terms of use: http://www.unicode.org/copyright.html
-/*
- **********************************************************************
- * Copyright (C) 2005-2013, International Business Machines
- * Corporation and others. All Rights Reserved.
- **********************************************************************
- */
-
-#include "unicode/utypes.h"
-
-#if !UCONFIG_NO_CONVERSION
-
-#include "csrucode.h"
-#include "csmatch.h"
-
-U_NAMESPACE_BEGIN
-
-CharsetRecog_Unicode::~CharsetRecog_Unicode()
-{
- // nothing to do
-}
-
-CharsetRecog_UTF_16_BE::~CharsetRecog_UTF_16_BE()
-{
- // nothing to do
-}
-
-const char *CharsetRecog_UTF_16_BE::getName() const
-{
- return "UTF-16BE";
-}
-
-// UTF-16 confidence calculation. Very simple minded, but better than nothing.
-// Any 8 bit non-control characters bump the confidence up. These have a zero high byte,
-// and are very likely to be UTF-16, although they could also be part of a UTF-32 code.
-// NULs are a contra-indication, they will appear commonly if the actual encoding is UTF-32.
-// NULs should be rare in actual text.
-
-static int32_t adjustConfidence(UChar codeUnit, int32_t confidence) {
- if (codeUnit == 0) {
- confidence -= 10;
- } else if ((codeUnit >= 0x20 && codeUnit <= 0xff) || codeUnit == 0x0a) {
- confidence += 10;
- }
- if (confidence < 0) {
- confidence = 0;
- } else if (confidence > 100) {
- confidence = 100;
- }
- return confidence;
-}
-
-
-UBool CharsetRecog_UTF_16_BE::match(InputText* textIn, CharsetMatch *results) const
-{
- const uint8_t *input = textIn->fRawInput;
- int32_t confidence = 10;
- int32_t length = textIn->fRawLength;
-
- int32_t bytesToCheck = (length > 30) ? 30 : length;
- for (int32_t charIndex=0; charIndex<bytesToCheck-1; charIndex+=2) {
- UChar codeUnit = (input[charIndex] << 8) | input[charIndex + 1];
- if (charIndex == 0 && codeUnit == 0xFEFF) {
- confidence = 100;
- break;
- }
- confidence = adjustConfidence(codeUnit, confidence);
- if (confidence == 0 || confidence == 100) {
- break;
- }
- }
- if (bytesToCheck < 4 && confidence < 100) {
- confidence = 0;
- }
- results->set(textIn, this, confidence);
- return (confidence > 0);
-}
-
-CharsetRecog_UTF_16_LE::~CharsetRecog_UTF_16_LE()
-{
- // nothing to do
-}
-
-const char *CharsetRecog_UTF_16_LE::getName() const
-{
- return "UTF-16LE";
-}
-
-UBool CharsetRecog_UTF_16_LE::match(InputText* textIn, CharsetMatch *results) const
-{
- const uint8_t *input = textIn->fRawInput;
- int32_t confidence = 10;
- int32_t length = textIn->fRawLength;
-
- int32_t bytesToCheck = (length > 30) ? 30 : length;
- for (int32_t charIndex=0; charIndex<bytesToCheck-1; charIndex+=2) {
- UChar codeUnit = input[charIndex] | (input[charIndex + 1] << 8);
- if (charIndex == 0 && codeUnit == 0xFEFF) {
- confidence = 100; // UTF-16 BOM
- if (length >= 4 && input[2] == 0 && input[3] == 0) {
- confidence = 0; // UTF-32 BOM
- }
- break;
- }
- confidence = adjustConfidence(codeUnit, confidence);
- if (confidence == 0 || confidence == 100) {
- break;
- }
- }
- if (bytesToCheck < 4 && confidence < 100) {
- confidence = 0;
- }
- results->set(textIn, this, confidence);
- return (confidence > 0);
-}
-
-CharsetRecog_UTF_32::~CharsetRecog_UTF_32()
-{
- // nothing to do
-}
-
-UBool CharsetRecog_UTF_32::match(InputText* textIn, CharsetMatch *results) const
-{
- const uint8_t *input = textIn->fRawInput;
- int32_t limit = (textIn->fRawLength / 4) * 4;
- int32_t numValid = 0;
- int32_t numInvalid = 0;
- bool hasBOM = FALSE;
- int32_t confidence = 0;
-
- if (limit > 0 && getChar(input, 0) == 0x0000FEFFUL) {
- hasBOM = TRUE;
- }
-
- for(int32_t i = 0; i < limit; i += 4) {
- int32_t ch = getChar(input, i);
-
- if (ch < 0 || ch >= 0x10FFFF || (ch >= 0xD800 && ch <= 0xDFFF)) {
- numInvalid += 1;
- } else {
- numValid += 1;
- }
- }
-
-
- // Cook up some sort of confidence score, based on presense of a BOM
- // and the existence of valid and/or invalid multi-byte sequences.
- if (hasBOM && numInvalid==0) {
- confidence = 100;
- } else if (hasBOM && numValid > numInvalid*10) {
- confidence = 80;
- } else if (numValid > 3 && numInvalid == 0) {
- confidence = 100;
- } else if (numValid > 0 && numInvalid == 0) {
- confidence = 80;
- } else if (numValid > numInvalid*10) {
- // Probably corruput UTF-32BE data. Valid sequences aren't likely by chance.
- confidence = 25;
- }
-
- results->set(textIn, this, confidence);
- return (confidence > 0);
-}
-
-CharsetRecog_UTF_32_BE::~CharsetRecog_UTF_32_BE()
-{
- // nothing to do
-}
-
-const char *CharsetRecog_UTF_32_BE::getName() const
-{
- return "UTF-32BE";
-}
-
-int32_t CharsetRecog_UTF_32_BE::getChar(const uint8_t *input, int32_t index) const
-{
- return input[index + 0] << 24 | input[index + 1] << 16 |
- input[index + 2] << 8 | input[index + 3];
-}
-
-CharsetRecog_UTF_32_LE::~CharsetRecog_UTF_32_LE()
-{
- // nothing to do
-}
-
-const char *CharsetRecog_UTF_32_LE::getName() const
-{
- return "UTF-32LE";
-}
-
-int32_t CharsetRecog_UTF_32_LE::getChar(const uint8_t *input, int32_t index) const
-{
- return input[index + 3] << 24 | input[index + 2] << 16 |
- input[index + 1] << 8 | input[index + 0];
-}
-
-U_NAMESPACE_END
-#endif