url: enforce valid UTF-8 in WHATWG parser

This commit implements the Web IDL USVString conversion, which mandates
all unpaired Unicode surrogates be turned into U+FFFD REPLACEMENT
CHARACTER. It also disallows Symbols to be used as USVString per spec.

Certain functions call into C++ methods in the binding that use the
Utf8Value class to access string arguments. Utf8Value already does the
normalization using V8's String::Write, so in those cases, instead of
doing the full USVString normalization, only a symbol check is done
(`'' + val`, which uses ES's ToString, versus `String()` which has
special provisions for symbols).

PR-URL: https://github.com/nodejs/node/pull/11436
Reviewed-By: Ben Noordhuis <info@bnoordhuis.nl>
Reviewed-By: James M Snell <jasnell@gmail.com>

1 files changed, 237 insertions, 0 deletions

diff --git a/test/fixtures/url-setter-tests-additional.js b/test/fixtures/url-setter-tests-additional.js
new file mode 100644
index 0000000000..b27ae336a2
--- /dev/null
+++ b/test/fixtures/url-setter-tests-additional.js
@@ -0,0 +1,237 @@
+module.exports = {
+  'username': [
+    {
+      'comment': 'Surrogate pair',
+      'href': 'https://github.com/',
+      'new_value': '\uD83D\uDE00',
+      'expected': {
+        'href': 'https://%F0%9F%98%80@github.com/',
+        'username': '%F0%9F%98%80'
+      }
+    },
+    {
+      'comment': 'Unpaired low surrogate 1',
+      'href': 'https://github.com/',
+      'new_value': '\uD83D',
+      'expected': {
+        'href': 'https://%EF%BF%BD@github.com/',
+        'username': '%EF%BF%BD'
+      }
+    },
+    {
+      'comment': 'Unpaired low surrogate 2',
+      'href': 'https://github.com/',
+      'new_value': '\uD83Dnode',
+      'expected': {
+        'href': 'https://%EF%BF%BDnode@github.com/',
+        'username': '%EF%BF%BDnode'
+      }
+    },
+    {
+      'comment': 'Unpaired high surrogate 1',
+      'href': 'https://github.com/',
+      'new_value': '\uDE00',
+      'expected': {
+        'href': 'https://%EF%BF%BD@github.com/',
+        'username': '%EF%BF%BD'
+      }
+    },
+    {
+      'comment': 'Unpaired high surrogate 2',
+      'href': 'https://github.com/',
+      'new_value': '\uDE00node',
+      'expected': {
+        'href': 'https://%EF%BF%BDnode@github.com/',
+        'username': '%EF%BF%BDnode'
+      }
+    }
+  ],
+  'password': [
+    {
+      'comment': 'Surrogate pair',
+      'href': 'https://github.com/',
+      'new_value': '\uD83D\uDE00',
+      'expected': {
+        'href': 'https://:%F0%9F%98%80@github.com/',
+        'password': '%F0%9F%98%80'
+      }
+    },
+    {
+      'comment': 'Unpaired low surrogate 1',
+      'href': 'https://github.com/',
+      'new_value': '\uD83D',
+      'expected': {
+        'href': 'https://:%EF%BF%BD@github.com/',
+        'password': '%EF%BF%BD'
+      }
+    },
+    {
+      'comment': 'Unpaired low surrogate 2',
+      'href': 'https://github.com/',
+      'new_value': '\uD83Dnode',
+      'expected': {
+        'href': 'https://:%EF%BF%BDnode@github.com/',
+        'password': '%EF%BF%BDnode'
+      }
+    },
+    {
+      'comment': 'Unpaired high surrogate 1',
+      'href': 'https://github.com/',
+      'new_value': '\uDE00',
+      'expected': {
+        'href': 'https://:%EF%BF%BD@github.com/',
+        'password': '%EF%BF%BD'
+      }
+    },
+    {
+      'comment': 'Unpaired high surrogate 2',
+      'href': 'https://github.com/',
+      'new_value': '\uDE00node',
+      'expected': {
+        'href': 'https://:%EF%BF%BDnode@github.com/',
+        'password': '%EF%BF%BDnode'
+      }
+    }
+  ],
+  'pathname': [
+    {
+      'comment': 'Surrogate pair',
+      'href': 'https://github.com/',
+      'new_value': '/\uD83D\uDE00',
+      'expected': {
+        'href': 'https://github.com/%F0%9F%98%80',
+        'pathname': '/%F0%9F%98%80'
+      }
+    },
+    {
+      'comment': 'Unpaired low surrogate 1',
+      'href': 'https://github.com/',
+      'new_value': '/\uD83D',
+      'expected': {
+        'href': 'https://github.com/%EF%BF%BD',
+        'pathname': '/%EF%BF%BD'
+      }
+    },
+    {
+      'comment': 'Unpaired low surrogate 2',
+      'href': 'https://github.com/',
+      'new_value': '/\uD83Dnode',
+      'expected': {
+        'href': 'https://github.com/%EF%BF%BDnode',
+        'pathname': '/%EF%BF%BDnode'
+      }
+    },
+    {
+      'comment': 'Unpaired high surrogate 1',
+      'href': 'https://github.com/',
+      'new_value': '/\uDE00',
+      'expected': {
+        'href': 'https://github.com/%EF%BF%BD',
+        'pathname': '/%EF%BF%BD'
+      }
+    },
+    {
+      'comment': 'Unpaired high surrogate 2',
+      'href': 'https://github.com/',
+      'new_value': '/\uDE00node',
+      'expected': {
+        'href': 'https://github.com/%EF%BF%BDnode',
+        'pathname': '/%EF%BF%BDnode'
+      }
+    }
+  ],
+  'search': [
+    {
+      'comment': 'Surrogate pair',
+      'href': 'https://github.com/',
+      'new_value': '\uD83D\uDE00',
+      'expected': {
+        'href': 'https://github.com/?%F0%9F%98%80',
+        'search': '?%F0%9F%98%80'
+      }
+    },
+    {
+      'comment': 'Unpaired low surrogate 1',
+      'href': 'https://github.com/',
+      'new_value': '\uD83D',
+      'expected': {
+        'href': 'https://github.com/?%EF%BF%BD',
+        'search': '?%EF%BF%BD'
+      }
+    },
+    {
+      'comment': 'Unpaired low surrogate 2',
+      'href': 'https://github.com/',
+      'new_value': '\uD83Dnode',
+      'expected': {
+        'href': 'https://github.com/?%EF%BF%BDnode',
+        'search': '?%EF%BF%BDnode'
+      }
+    },
+    {
+      'comment': 'Unpaired high surrogate 1',
+      'href': 'https://github.com/',
+      'new_value': '\uDE00',
+      'expected': {
+        'href': 'https://github.com/?%EF%BF%BD',
+        'search': '?%EF%BF%BD'
+      }
+    },
+    {
+      'comment': 'Unpaired high surrogate 2',
+      'href': 'https://github.com/',
+      'new_value': '\uDE00node',
+      'expected': {
+        'href': 'https://github.com/?%EF%BF%BDnode',
+        'search': '?%EF%BF%BDnode'
+      }
+    }
+  ],
+  'hash': [
+    {
+      'comment': 'Surrogate pair',
+      'href': 'https://github.com/',
+      'new_value': '\uD83D\uDE00',
+      'expected': {
+        'href': 'https://github.com/#%F0%9F%98%80',
+        'hash': '#%F0%9F%98%80'
+      }
+    },
+    {
+      'comment': 'Unpaired low surrogate 1',
+      'href': 'https://github.com/',
+      'new_value': '\uD83D',
+      'expected': {
+        'href': 'https://github.com/#%EF%BF%BD',
+        'hash': '#%EF%BF%BD'
+      }
+    },
+    {
+      'comment': 'Unpaired low surrogate 2',
+      'href': 'https://github.com/',
+      'new_value': '\uD83Dnode',
+      'expected': {
+        'href': 'https://github.com/#%EF%BF%BDnode',
+        'hash': '#%EF%BF%BDnode'
+      }
+    },
+    {
+      'comment': 'Unpaired high surrogate 1',
+      'href': 'https://github.com/',
+      'new_value': '\uDE00',
+      'expected': {
+        'href': 'https://github.com/#%EF%BF%BD',
+        'hash': '#%EF%BF%BD'
+      }
+    },
+    {
+      'comment': 'Unpaired high surrogate 2',
+      'href': 'https://github.com/',
+      'new_value': '\uDE00node',
+      'expected': {
+        'href': 'https://github.com/#%EF%BF%BDnode',
+        'hash': '#%EF%BF%BDnode'
+      }
+    }
+  ]
+};