url: update WHATWG URL API to latest spec

- Update to spec
  - Add opaque hosts
  - File state did not correctly deal with lack of base URL
  - Cleanup API for file and non-special URLs
  - Allow % and IPv6 addresses in non-special URL hosts
  - Use specific names for percent-encode sets
  - Add empty host concept for file and non-special URLs
  - Clarify IPv6 serializer
- Fix existing mistakes
  - Add missing ':' to forbidden host code point list.
  - Correct IPv4 parser empty label behavior
- Maintain type equivalence in URLContext with spec
  - scheme, username, and password should always be strings
  - host, port, query, and fragment may be strings or null
  - Align scheme state more closely with the spec
  - Make sure the `special` variable is always synced with
    URL_FLAG_SPECIAL.

PR-URL: https://github.com/nodejs/node/pull/12523
Fixes: https://github.com/nodejs/node/issues/10608
Fixes: https://github.com/nodejs/node/issues/10634
Refs: https://github.com/whatwg/url/pull/185
Refs: https://github.com/whatwg/url/pull/225
Refs: https://github.com/whatwg/url/pull/224
Refs: https://github.com/whatwg/url/pull/218
Refs: https://github.com/whatwg/url/pull/243
Refs: https://github.com/whatwg/url/pull/260
Refs: https://github.com/whatwg/url/pull/268
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Daijiro Wachi <daijiro.wachi@gmail.com>
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>

1 files changed, 84 insertions, 95 deletions

diff --git a/lib/internal/url.js b/lib/internal/url.js
index 5855a47ea4..cacdff7c3a 100644
--- a/lib/internal/url.js
+++ b/lib/internal/url.js
@@ -8,6 +8,8 @@ const {
 const binding = process.binding('url');
 const context = Symbol('context');
 const cannotBeBase = Symbol('cannot-be-base');
+const cannotHaveUsernamePasswordPort =
+    Symbol('cannot-have-username-password-port');
 const special = Symbol('special');
 const searchParams = Symbol('query');
 const querystring = require('querystring');
@@ -42,7 +44,7 @@ const kOpaqueOrigin = 'null';
 // - https://html.spec.whatwg.org/multipage/browsers.html#ascii-serialisation-of-an-origin
 function serializeTupleOrigin(scheme, host, port, unicode = true) {
   const unicodeHost = unicode ? domainToUnicode(host) : host;
-  return `${scheme}//${unicodeHost}${port == null ? '' : `:${port}`}`;
+  return `${scheme}//${unicodeHost}${port === null ? '' : `:${port}`}`;
 }
 
 // This class provides the internal state of a URL object. An instance of this
@@ -54,14 +56,14 @@ function serializeTupleOrigin(scheme, host, port, unicode = true) {
 class URLContext {
   constructor() {
     this.flags = 0;
-    this.scheme = undefined;
-    this.username = undefined;
-    this.password = undefined;
-    this.host = undefined;
-    this.port = undefined;
+    this.scheme = ':';
+    this.username = '';
+    this.password = '';
+    this.host = null;
+    this.port = null;
     this.path = [];
-    this.query = undefined;
-    this.fragment = undefined;
+    this.query = null;
+    this.fragment = null;
   }
 }
 
@@ -70,10 +72,10 @@ function onParseComplete(flags, protocol, username, password,
   var ctx = this[context];
   ctx.flags = flags;
   ctx.scheme = protocol;
-  ctx.username = username;
-  ctx.password = password;
+  ctx.username = (flags & binding.URL_FLAGS_HAS_USERNAME) !== 0 ? username : '';
+  ctx.password = (flags & binding.URL_FLAGS_HAS_PASSWORD) !== 0 ? password : '';
   ctx.port = port;
-  ctx.path = path;
+  ctx.path = (flags & binding.URL_FLAGS_HAS_PATH) !== 0 ? path : [];
   ctx.query = query;
   ctx.fragment = fragment;
   ctx.host = host;
@@ -101,52 +103,37 @@ function parse(url, input, base) {
 
 function onParseProtocolComplete(flags, protocol, username, password,
                                  host, port, path, query, fragment) {
-  const newIsSpecial = (flags & binding.URL_FLAGS_SPECIAL) !== 0;
-  const s = this[special];
   const ctx = this[context];
-  if ((s && !newIsSpecial) || (!s && newIsSpecial)) {
-    return;
-  }
-  if (protocol === 'file:' &&
-    (ctx.username || ctx.password || ctx.port !== undefined)) {
-    return;
-  }
-  if (ctx.scheme === 'file:' && !ctx.host) {
-    return;
-  }
-  if (newIsSpecial) {
+  if ((flags & binding.URL_FLAGS_SPECIAL) !== 0) {
     ctx.flags |= binding.URL_FLAGS_SPECIAL;
   } else {
     ctx.flags &= ~binding.URL_FLAGS_SPECIAL;
   }
-  if (protocol) {
-    ctx.scheme = protocol;
-    ctx.flags |= binding.URL_FLAGS_HAS_SCHEME;
-  } else {
-    ctx.flags &= ~binding.URL_FLAGS_HAS_SCHEME;
-  }
+  ctx.scheme = protocol;
 }
 
 function onParseHostComplete(flags, protocol, username, password,
                              host, port, path, query, fragment) {
   const ctx = this[context];
-  if (host) {
+  if ((flags & binding.URL_FLAGS_HAS_HOST) !== 0) {
     ctx.host = host;
     ctx.flags |= binding.URL_FLAGS_HAS_HOST;
   } else {
+    ctx.host = null;
     ctx.flags &= ~binding.URL_FLAGS_HAS_HOST;
   }
-  if (port !== undefined)
+  if (port !== null)
     ctx.port = port;
 }
 
 function onParseHostnameComplete(flags, protocol, username, password,
                                  host, port, path, query, fragment) {
   const ctx = this[context];
-  if (host) {
+  if ((flags & binding.URL_FLAGS_HAS_HOST) !== 0) {
     ctx.host = host;
     ctx.flags |= binding.URL_FLAGS_HAS_HOST;
   } else {
+    ctx.host = null;
     ctx.flags &= ~binding.URL_FLAGS_HAS_HOST;
   }
 }
@@ -159,29 +146,29 @@ function onParsePortComplete(flags, protocol, username, password,
 function onParsePathComplete(flags, protocol, username, password,
                              host, port, path, query, fragment) {
   const ctx = this[context];
-  if (path) {
+  if ((flags & binding.URL_FLAGS_HAS_PATH) !== 0) {
     ctx.path = path;
     ctx.flags |= binding.URL_FLAGS_HAS_PATH;
   } else {
+    ctx.path = [];
     ctx.flags &= ~binding.URL_FLAGS_HAS_PATH;
   }
+
+  // The C++ binding may set host to empty string.
+  if ((flags & binding.URL_FLAGS_HAS_HOST) !== 0) {
+    ctx.host = host;
+    ctx.flags |= binding.URL_FLAGS_HAS_HOST;
+  }
 }
 
 function onParseSearchComplete(flags, protocol, username, password,
                                host, port, path, query, fragment) {
-  const ctx = this[context];
-  ctx.query = query;
+  this[context].query = query;
 }
 
 function onParseHashComplete(flags, protocol, username, password,
                              host, port, path, query, fragment) {
-  const ctx = this[context];
-  if (fragment) {
-    ctx.fragment = fragment;
-    ctx.flags |= binding.URL_FLAGS_HAS_FRAGMENT;
-  } else {
-    ctx.flags &= ~binding.URL_FLAGS_HAS_FRAGMENT;
-  }
+  this[context].fragment = fragment;
 }
 
 function getEligibleConstructor(obj) {
@@ -214,6 +201,14 @@ class URL {
     return (this[context].flags & binding.URL_FLAGS_CANNOT_BE_BASE) !== 0;
   }
 
+  // https://url.spec.whatwg.org/#cannot-have-a-username-password-port
+  get [cannotHaveUsernamePasswordPort]() {
+    const { host, scheme } = this[context];
+    return ((host == null || host === '') ||
+            this[cannotBeBase] ||
+            scheme === 'file:');
+  }
+
   [util.inspect.custom](depth, opts) {
     if (this == null ||
         Object.getPrototypeOf(this[context]) !== URLContext.prototype) {
@@ -235,7 +230,7 @@ class URL {
     obj.origin = this.origin;
     obj.protocol = this.protocol;
     obj.username = this.username;
-    obj.password = (opts.showHidden || ctx.password == null) ?
+    obj.password = (opts.showHidden || ctx.password === '') ?
                      this.password : '--------';
     obj.host = this.host;
     obj.hostname = this.hostname;
@@ -270,14 +265,11 @@ Object.defineProperties(URL.prototype, {
         auth: true
       }, options);
       const ctx = this[context];
-      var ret;
-      if (this.protocol)
-        ret = this.protocol;
-      if (ctx.host !== undefined) {
+      var ret = ctx.scheme;
+      if (ctx.host !== null) {
         ret += '//';
-        const has_username = typeof ctx.username === 'string';
-        const has_password = typeof ctx.password === 'string' &&
-                             ctx.password !== '';
+        const has_username = ctx.username !== '';
+        const has_password = ctx.password !== '';
         if (options.auth && (has_username || has_password)) {
           if (has_username)
             ret += ctx.username;
@@ -292,9 +284,9 @@ Object.defineProperties(URL.prototype, {
       }
       if (this.pathname)
         ret += this.pathname;
-      if (options.search && typeof ctx.query === 'string')
+      if (options.search && ctx.query !== null)
         ret += `?${ctx.query}`;
-      if (options.fragment && typeof ctx.fragment === 'string')
+      if (options.fragment && ctx.fragment !== null)
         ret += `#${ctx.fragment}`;
       return ret;
     }
@@ -363,7 +355,12 @@ Object.defineProperties(URL.prototype, {
       scheme = `${scheme}`;
       if (scheme.length === 0)
         return;
-      binding.parse(scheme, binding.kSchemeStart, null, this[context],
+      const ctx = this[context];
+      if (ctx.scheme === 'file:' &&
+          (ctx.host === '' || ctx.host === null)) {
+        return;
+      }
+      binding.parse(scheme, binding.kSchemeStart, null, ctx,
                     onParseProtocolComplete.bind(this));
     }
   },
@@ -371,16 +368,16 @@ Object.defineProperties(URL.prototype, {
     enumerable: true,
     configurable: true,
     get() {
-      return this[context].username || '';
+      return this[context].username;
     },
     set(username) {
       // toUSVString is not needed.
       username = `${username}`;
-      if (!this.hostname)
+      if (this[cannotHaveUsernamePasswordPort])
         return;
       const ctx = this[context];
-      if (!username) {
-        ctx.username = null;
+      if (username === '') {
+        ctx.username = '';
         ctx.flags &= ~binding.URL_FLAGS_HAS_USERNAME;
         return;
       }
@@ -392,16 +389,16 @@ Object.defineProperties(URL.prototype, {
     enumerable: true,
     configurable: true,
     get() {
-      return this[context].password || '';
+      return this[context].password;
     },
     set(password) {
       // toUSVString is not needed.
       password = `${password}`;
-      if (!this.hostname)
+      if (this[cannotHaveUsernamePasswordPort])
         return;
       const ctx = this[context];
-      if (!password) {
-        ctx.password = null;
+      if (password === '') {
+        ctx.password = '';
         ctx.flags &= ~binding.URL_FLAGS_HAS_PASSWORD;
         return;
       }
@@ -415,7 +412,7 @@ Object.defineProperties(URL.prototype, {
     get() {
       const ctx = this[context];
       var ret = ctx.host || '';
-      if (ctx.port !== undefined)
+      if (ctx.port !== null)
         ret += `:${ctx.port}`;
       return ret;
     },
@@ -423,15 +420,8 @@ Object.defineProperties(URL.prototype, {
       const ctx = this[context];
       // toUSVString is not needed.
       host = `${host}`;
-      if (this[cannotBeBase] ||
-          (this[special] && host.length === 0)) {
-        // Cannot set the host if cannot-be-base is set or
-        // scheme is special and host length is zero
-        return;
-      }
-      if (!host) {
-        ctx.host = null;
-        ctx.flags &= ~binding.URL_FLAGS_HAS_HOST;
+      if (this[cannotBeBase]) {
+        // Cannot set the host if cannot-be-base is set
         return;
       }
       binding.parse(host, binding.kHost, null, ctx,
@@ -448,15 +438,8 @@ Object.defineProperties(URL.prototype, {
       const ctx = this[context];
       // toUSVString is not needed.
       host = `${host}`;
-      if (this[cannotBeBase] ||
-          (this[special] && host.length === 0)) {
-        // Cannot set the host if cannot-be-base is set or
-        // scheme is special and host length is zero
-        return;
-      }
-      if (!host) {
-        ctx.host = null;
-        ctx.flags &= ~binding.URL_FLAGS_HAS_HOST;
+      if (this[cannotBeBase]) {
+        // Cannot set the host if cannot-be-base is set
         return;
       }
       binding.parse(host, binding.kHostname, null, ctx,
@@ -468,17 +451,16 @@ Object.defineProperties(URL.prototype, {
     configurable: true,
     get() {
       const port = this[context].port;
-      return port === undefined ? '' : String(port);
+      return port === null ? '' : String(port);
     },
     set(port) {
       // toUSVString is not needed.
       port = `${port}`;
-      const ctx = this[context];
-      if (!ctx.host || this[cannotBeBase] ||
-          this.protocol === 'file:')
+      if (this[cannotHaveUsernamePasswordPort])
         return;
+      const ctx = this[context];
       if (port === '') {
-        ctx.port = undefined;
+        ctx.port = null;
         return;
       }
       binding.parse(port, binding.kPort, null, ctx,
@@ -492,7 +474,9 @@ Object.defineProperties(URL.prototype, {
       const ctx = this[context];
       if (this[cannotBeBase])
         return ctx.path[0];
-      return ctx.path !== undefined ? `/${ctx.path.join('/')}` : '';
+      if (ctx.path.length === 0)
+        return '';
+      return `/${ctx.path.join('/')}`;
     },
     set(path) {
       // toUSVString is not needed.
@@ -507,13 +491,15 @@ Object.defineProperties(URL.prototype, {
     enumerable: true,
     configurable: true,
     get() {
-      const ctx = this[context];
-      return !ctx.query ? '' : `?${ctx.query}`;
+      const { query } = this[context];
+      if (query === null || query === '')
+        return '';
+      return `?${query}`;
     },
     set(search) {
       const ctx = this[context];
       search = toUSVString(search);
-      if (!search) {
+      if (search === '') {
         ctx.query = null;
         ctx.flags &= ~binding.URL_FLAGS_HAS_QUERY;
       } else {
@@ -539,8 +525,10 @@ Object.defineProperties(URL.prototype, {
     enumerable: true,
     configurable: true,
     get() {
-      const ctx = this[context];
-      return !ctx.fragment ? '' : `#${ctx.fragment}`;
+      const { fragment } = this[context];
+      if (fragment === null || fragment === '')
+        return '';
+      return `#${fragment}`;
     },
     set(hash) {
       const ctx = this[context];
@@ -553,6 +541,7 @@ Object.defineProperties(URL.prototype, {
       }
       if (hash[0] === '#') hash = hash.slice(1);
       ctx.fragment = '';
+      ctx.flags |= binding.URL_FLAGS_HAS_FRAGMENT;
       binding.parse(hash, binding.kFragment, null, ctx,
                     onParseHashComplete.bind(this));
     }
@@ -1384,10 +1373,10 @@ function constructUrl(flags, protocol, username, password,
   var ctx = new URLContext();
   ctx.flags = flags;
   ctx.scheme = protocol;
-  ctx.username = username;
-  ctx.password = password;
+  ctx.username = (flags & binding.URL_FLAGS_HAS_USERNAME) !== 0 ? username : '';
+  ctx.password = (flags & binding.URL_FLAGS_HAS_PASSWORD) !== 0 ? password : '';
   ctx.port = port;
-  ctx.path = path;
+  ctx.path = (flags & binding.URL_FLAGS_HAS_PATH) !== 0 ? path : [];
   ctx.query = query;
   ctx.fragment = fragment;
   ctx.host = host;