crypto: fix UB in computing max message size

Before this commit it computed `(1<<(8*(15-iv_len)))-1` for `iv_len>=11`
and that reduces to `(1<<32)-1` for `iv_len==11`.  Left-shifting past
the sign bit and overflowing a signed integral type are both undefined
behaviors.

This commit switches to fixed values and restricts the `iv_len==11`
case to `INT_MAX`, as was already the case for all `iv_len<=10`.

PR-URL: https://github.com/nodejs/node/pull/21462
Reviewed-By: Colin Ihrig <cjihrig@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Tobias Nießen <tniessen@tnie.de>

1 files changed, 10 insertions, 0 deletions

diff --git a/test/parallel/test-crypto-authenticated.js b/test/parallel/test-crypto-authenticated.js
index ee91e31e9c..a4fe105b7e 100644
--- a/test/parallel/test-crypto-authenticated.js
+++ b/test/parallel/test-crypto-authenticated.js
@@ -1016,3 +1016,13 @@ for (const test of TEST_CASES) {
   assert.strictEqual(decrypt.update('807022', 'hex', 'hex'), 'abcdef');
   assert.strictEqual(decrypt.final('hex'), '');
 }
+
+// Test that an IV length of 11 does not overflow max_message_size_.
+{
+  const key = 'x'.repeat(16);
+  const iv = Buffer.from('112233445566778899aabb', 'hex');
+  const options = { authTagLength: 8 };
+  const encrypt = crypto.createCipheriv('aes-128-ccm', key, iv, options);
+  encrypt.update('boom');  // Should not throw 'Message exceeds maximum size'.
+  encrypt.final();
+}