crypto: throw proper errors if out enc is UTF-16

Throw `Error`s instead of hard crashing when the `.digest()` output
encoding is UTF-16.

Fixes: https://github.com/nodejs/node/issues/9817
PR-URL: https://github.com/nodejs/node/pull/12752
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Colin Ihrig <cjihrig@gmail.com>
Reviewed-By: Refael Ackermann <refack@gmail.com>
Reviewed-By: Gibson Fahnestock <gibfahn@gmail.com>

1 files changed, 8 insertions, 0 deletions

diff --git a/src/node_crypto.cc b/src/node_crypto.cc
index ee48163380..361f1a0005 100644
--- a/src/node_crypto.cc
+++ b/src/node_crypto.cc
@@ -3798,6 +3798,10 @@ void Hmac::HmacDigest(const FunctionCallbackInfo<Value>& args) {
     encoding = ParseEncoding(env->isolate(), args[0], BUFFER);
   }
 
+  if (encoding == UCS2) {
+    return env->ThrowError("hmac.digest() does not support UTF-16");
+  }
+
   unsigned char* md_value = nullptr;
   unsigned int md_len = 0;
 
@@ -3921,6 +3925,10 @@ void Hash::HashDigest(const FunctionCallbackInfo<Value>& args) {
     encoding = ParseEncoding(env->isolate(), args[0], BUFFER);
   }
 
+  if (encoding == UCS2) {
+    return env->ThrowError("hash.digest() does not support UTF-16");
+  }
+
   unsigned char md_value[EVP_MAX_MD_SIZE];
   unsigned int md_len;