From 061f2075cf81017cdb40de80533ba18746743c94 Mon Sep 17 00:00:00 2001 From: isaacs Date: Thu, 11 Oct 2012 15:53:11 -0700 Subject: string_decoder: Add 'end' method, do base64 properly --- doc/api/string_decoder.markdown | 8 +++- lib/fs.js | 5 +++ lib/http.js | 10 +++++ lib/net.js | 5 +++ lib/string_decoder.js | 26 ++++++++++++ lib/tls.js | 5 +++ test/simple/test-string-decoder-end.js | 75 ++++++++++++++++++++++++++++++++++ 7 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 test/simple/test-string-decoder-end.js diff --git a/doc/api/string_decoder.markdown b/doc/api/string_decoder.markdown index 2ad2fb1e4b..b8a6252d66 100644 --- a/doc/api/string_decoder.markdown +++ b/doc/api/string_decoder.markdown @@ -19,6 +19,10 @@ additional support for utf8. Accepts a single argument, `encoding` which defaults to `utf8`. -### StringDecoder.write(buffer) +### decoder.write(buffer) -Returns a decoded string. \ No newline at end of file +Returns a decoded string. + +### decoder.end() + +Returns any trailing bytes that were left in the buffer. diff --git a/lib/fs.js b/lib/fs.js index e8a9aa4f72..8041ab620d 100644 --- a/lib/fs.js +++ b/lib/fs.js @@ -1402,6 +1402,11 @@ ReadStream.prototype._read = function() { } if (bytesRead === 0) { + if (this._decoder) { + var ret = this._decoder.end(); + if (ret) + this.emit('data', ret); + } self.emit('end'); self.destroy(); return; diff --git a/lib/http.js b/lib/http.js index 4aa8f5cb40..3c8b60944c 100644 --- a/lib/http.js +++ b/lib/http.js @@ -363,6 +363,11 @@ IncomingMessage.prototype._emitData = function(d) { IncomingMessage.prototype._emitEnd = function() { if (!this._endEmitted) { + if (this._decoder) { + var ret = this._decoder.end(); + if (ret) + this.emit('data', ret); + } this.emit('end'); } @@ -1859,6 +1864,11 @@ Client.prototype.request = function(method, path, headers) { // but it will get removed when we remove this legacy interface. c.on('socket', function(s) { s.on('end', function() { + if (self._decoder) { + var ret = self._decoder.end(); + if (ret) + self.emit('data', ret); + } self.emit('end'); }); }); diff --git a/lib/net.js b/lib/net.js index 0decfe963c..46d7d0b173 100644 --- a/lib/net.js +++ b/lib/net.js @@ -414,6 +414,11 @@ function onread(buffer, offset, length) { if (!self.writable) self._destroy(); if (!self.allowHalfOpen) self.end(); + if (self._decoder) { + var ret = self._decoder.end(); + if (ret) + self.emit('data', ret); + } if (self._events && self._events['end']) self.emit('end'); if (self.onend) self.onend(); } else { diff --git a/lib/string_decoder.js b/lib/string_decoder.js index 879e590647..31d4b24702 100644 --- a/lib/string_decoder.js +++ b/lib/string_decoder.js @@ -32,6 +32,11 @@ var StringDecoder = exports.StringDecoder = function(encoding) { this.surrogateSize = 2; this.detectIncompleteChar = utf16DetectIncompleteChar; break; + case 'base64': + // Base-64 stores 3 bytes in 4 chars, and pads the remainder. + this.surrogateSize = 3; + this.detectIncompleteChar = base64DetectIncompleteChar; + break; default: this.write = passThroughWrite; return; @@ -145,6 +150,21 @@ StringDecoder.prototype.detectIncompleteChar = function(buffer) { return i; }; +StringDecoder.prototype.end = function(buffer) { + var res = ''; + if (buffer && buffer.length) + res = this.write(buffer); + + if (this.charReceived) { + var cr = this.charReceived; + var buf = this.charBuffer; + var enc = this.encoding; + res += buf.slice(0, cr).toString(enc); + } + + return res; +}; + function passThroughWrite(buffer) { return buffer.toString(this.encoding); } @@ -154,3 +174,9 @@ function utf16DetectIncompleteChar(buffer) { this.charLength = incomplete ? 2 : 0; return incomplete; } + +function base64DetectIncompleteChar(buffer) { + var incomplete = this.charReceived = buffer.length % 3; + this.charLength = incomplete ? 3 : 0; + return incomplete; +} diff --git a/lib/tls.js b/lib/tls.js index 1fe4f78999..ed6c5a7d44 100644 --- a/lib/tls.js +++ b/lib/tls.js @@ -967,6 +967,11 @@ SecurePair.prototype.destroy = function() { self.cleartext.writable = self.cleartext.readable = false; process.nextTick(function() { + if (self.cleartext._decoder) { + var ret = self.cleartext._decoder.end(); + if (ret) + self.cleartext.emit('data', ret); + } self.cleartext.emit('end'); self.encrypted.emit('close'); self.cleartext.emit('close'); diff --git a/test/simple/test-string-decoder-end.js b/test/simple/test-string-decoder-end.js new file mode 100644 index 0000000000..fea55d410b --- /dev/null +++ b/test/simple/test-string-decoder-end.js @@ -0,0 +1,75 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +// verify that the string decoder works getting 1 byte at a time, +// the whole buffer at once, and that both match the .toString(enc) +// result of the entire buffer. + +var assert = require('assert'); +var SD = require('string_decoder').StringDecoder; +var encodings = ['base64', 'hex', 'utf8', 'utf16le', 'ucs2']; + +var bufs = [ '☃💩', 'asdf' ].map(function(b) { + return new Buffer(b); +}); + +// also test just arbitrary bytes from 0-15. +for (var i = 1; i <= 16; i++) { + var bytes = new Array(i).join('.').split('.').map(function(_, j) { + return j + 0x78; + }); + bufs.push(new Buffer(bytes)); +} + +encodings.forEach(testEncoding); + +console.log('ok'); + +function testEncoding(encoding) { + bufs.forEach(function(buf) { + testBuf(encoding, buf); + }); +} + +function testBuf(encoding, buf) { + console.error('# %s', encoding, buf); + + // write one byte at a time. + var s = new SD(encoding); + var res1 = ''; + for (var i = 0; i < buf.length; i++) { + res1 += s.write(buf.slice(i, i + 1)); + } + res1 += s.end(); + + // write the whole buffer at once. + var res2 = ''; + var s = new SD(encoding); + res2 += s.write(buf); + res2 += s.end(); + + // .toString() on the buffer + var res3 = buf.toString(encoding); + + console.log('expect=%j', res3); + assert.equal(res1, res3, 'one byte at a time should match toString'); + assert.equal(res2, res3, 'all bytes at once should match toString'); +} -- cgit v1.2.3