diff options
author | Felix Geisendörfer <felix@debuggable.com> | 2014-05-13 17:42:48 +0200 |
---|---|---|
committer | Timothy J Fontaine <tjfontaine@gmail.com> | 2014-06-06 15:07:29 -0700 |
commit | 80eff968292ab1024efbba6f55114ae2bba66579 (patch) | |
tree | 86c444119eabea14b167de2cd0f2bf8af53d0cbe /lib/string_decoder.js | |
parent | 9fbd0f0f7df16536533c331e3c634b203087d521 (diff) | |
download | android-node-v8-80eff968292ab1024efbba6f55114ae2bba66579.tar.gz android-node-v8-80eff968292ab1024efbba6f55114ae2bba66579.tar.bz2 android-node-v8-80eff968292ab1024efbba6f55114ae2bba66579.zip |
string_decoder: Add more comments
Diffstat (limited to 'lib/string_decoder.js')
-rw-r--r-- | lib/string_decoder.js | 25 |
1 files changed, 25 insertions, 0 deletions
diff --git a/lib/string_decoder.js b/lib/string_decoder.js index fe4c9fc170..4a458334ec 100644 --- a/lib/string_decoder.js +++ b/lib/string_decoder.js @@ -25,6 +25,14 @@ function assertEncoding(encoding) { } } +// StringDecoder provides an interface for efficiently splitting a series of +// buffers into a series of JS strings without breaking apart multi-byte +// characters. CESU-8 is handled as part of the UTF-8 encoding. +// +// @TODO Handling all encodings inside a single object makes it very difficult +// to reason about this code, so it should be split up in the future. +// @TODO There should be a utf8-strict encoding that rejects invalid UTF-8 code +// points as used by CESU-8. var StringDecoder = exports.StringDecoder = function(encoding) { this.encoding = (encoding || 'utf8').toLowerCase().replace(/[-_]/, ''); assertEncoding(encoding); @@ -49,12 +57,25 @@ var StringDecoder = exports.StringDecoder = function(encoding) { return; } + // Enough space to store all bytes of a single character. UTF-8 needs 4 + // bytes, but CESU-8 may require up to 6 (3 bytes per surrogate). this.charBuffer = new Buffer(6); + // Number of bytes received for the current incomplete multi-byte character. this.charReceived = 0; + // Number of bytes expected for the current incomplete multi-byte character. this.charLength = 0; }; +// write decodes the given buffer and returns it as JS string that is +// guaranteed to not contain any partial multi-byte characters. Any partial +// character found at the end of the buffer is buffered up, and will be +// returned when calling write again with the remaining bytes. +// +// Note: Converting a Buffer containing an orphan surrogate to a String +// currently works, but converting a String to a Buffer (via `new Buffer`, or +// Buffer#write) will replace incomplete surrogates with the unicode +// replacement character. See https://codereview.chromium.org/121173009/ . StringDecoder.prototype.write = function(buffer) { var charStr = ''; // if our last write ended with an incomplete multibyte character @@ -123,6 +144,10 @@ StringDecoder.prototype.write = function(buffer) { return charStr; }; +// detectIncompleteChar determines if there is an incomplete UTF-8 character at +// the end of the given buffer. If so, it sets this.charLength to the byte +// length that character, and sets this.charReceived to the number of bytes +// that are available for this character. StringDecoder.prototype.detectIncompleteChar = function(buffer) { // determine how many bytes we have to check at the end of this buffer var i = (buffer.length >= 3) ? 3 : buffer.length; |