fs: reduce memory retention when streaming small files

Fixes: https://github.com/nodejs/node/issues/21967

PR-URL: https://github.com/nodejs/node/pull/21968
Reviewed-By: Сковорода Никита Андреевич <chalkerx@gmail.com>
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>
Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>

1 files changed, 18 insertions, 1 deletions

diff --git a/lib/internal/fs/streams.js b/lib/internal/fs/streams.js
index f527b1de4b..92bd9a4c15 100644
--- a/lib/internal/fs/streams.js
+++ b/lib/internal/fs/streams.js
@@ -21,9 +21,18 @@ const util = require('util');
 const kMinPoolSpace = 128;
 
 let pool;
+// It can happen that we expect to read a large chunk of data, and reserve
+// a large chunk of the pool accordingly, but the read() call only filled
+// a portion of it. If a concurrently executing read() then uses the same pool,
+// the "reserved" portion cannot be used, so we allow it to be re-used as a
+// new pool later.
+const poolFragments = [];
 
 function allocNewPool(poolSize) {
-  pool = Buffer.allocUnsafe(poolSize);
+  if (poolFragments.length > 0)
+    pool = poolFragments.pop();
+  else
+    pool = Buffer.allocUnsafe(poolSize);
   pool.used = 0;
 }
 
@@ -171,6 +180,14 @@ ReadStream.prototype._read = function(n) {
       this.emit('error', er);
     } else {
       let b = null;
+      // Now that we know how much data we have actually read, re-wind the
+      // 'used' field if we can, and otherwise allow the remainder of our
+      // reservation to be used as a new pool later.
+      if (start + toRead === thisPool.used && thisPool === pool)
+        thisPool.used += bytesRead - toRead;
+      else if (toRead - bytesRead > kMinPoolSpace)
+        poolFragments.push(thisPool.slice(start + bytesRead, start + toRead));
+
       if (bytesRead > 0) {
         this.bytesRead += bytesRead;
         b = thisPool.slice(start, start + bytesRead);