commit 53c79a44c78f35068f3895765bff47e90646b4d9
parent 6d2bf179e48ab4c6de1b95bd180df250461fb0d6
Author: Florian Dold <dold@taler.net>
Date: Fri, 31 Jul 2026 11:54:11 +0200
util: fix the URL parser spec derivations
Diffstat:
2 files changed, 197 insertions(+), 16 deletions(-)
diff --git a/packages/taler-util/src/whatwg-url.test.ts b/packages/taler-util/src/whatwg-url.test.ts
@@ -0,0 +1,130 @@
+/*
+ This file is part of GNU Taler
+ (C) 2026 Taler Systems S.A.
+
+ GNU Taler is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+ */
+
+import assert from "node:assert";
+import { test } from "node:test";
+import { URL } from "./url.js";
+
+// The setters below all re-run the URL parser with a state override, which
+// tells it to update one component and stop instead of continuing into the
+// next parser state. Without that flag the parser falls through, most
+// visibly by appending an empty path segment.
+
+test("the hostname setter leaves the path alone", (t) => {
+ const root = new URL("https://example.com/");
+ root.hostname = "exchange.example.com";
+ assert.strictEqual(root.href, "https://exchange.example.com/");
+
+ const withPath = new URL("https://example.com/foo/bar");
+ withPath.hostname = "x.example.com";
+ assert.strictEqual(withPath.href, "https://x.example.com/foo/bar");
+});
+
+test("the hostname setter keeps the port", (t) => {
+ const u = new URL("https://example.com:8081/");
+ u.hostname = "exchange.example.com";
+ assert.strictEqual(u.href, "https://exchange.example.com:8081/");
+});
+
+test("the host setter takes host and port together", (t) => {
+ const u = new URL("https://example.com/");
+ u.host = "x.example.com:99";
+ assert.strictEqual(u.href, "https://x.example.com:99/");
+});
+
+test("the port setter can clear the port", (t) => {
+ const u = new URL("https://example.com:8081/");
+ u.port = "";
+ assert.strictEqual(u.href, "https://example.com/");
+
+ const other = new URL("https://example.com/");
+ other.port = "1234";
+ assert.strictEqual(other.href, "https://example.com:1234/");
+});
+
+test("the protocol setter replaces only the scheme", (t) => {
+ const u = new URL("https://example.com/foo");
+ u.protocol = "http:";
+ assert.strictEqual(u.href, "http://example.com/foo");
+});
+
+test("the pathname setter replaces the whole path", (t) => {
+ const u = new URL("https://example.com/a/b");
+ u.pathname = "/c";
+ assert.strictEqual(u.href, "https://example.com/c");
+});
+
+test("setting a component twice is idempotent", (t) => {
+ const u = new URL("https://example.com/");
+ u.hostname = "exchange.example.com";
+ u.hostname = "exchange.example.com";
+ assert.strictEqual(u.href, "https://exchange.example.com/");
+});
+
+// The cases below are taken from the Web Platform Tests URL corpus
+// (url/resources/urltestdata.json).
+
+test("an ASCII host is lowercased", (t) => {
+ assert.strictEqual(
+ new URL("http://ExAmPlE.CoM", "http://other.com/").href,
+ "http://example.com/",
+ );
+ assert.strictEqual(
+ new URL("http://a.b.c.XN--pokxncvks").host,
+ "a.b.c.xn--pokxncvks",
+ );
+ assert.strictEqual(new URL("http://0X7F.0.0.0X7G").host, "0x7f.0.0.0x7g");
+});
+
+test("a scheme-only URL keeps an empty opaque path", (t) => {
+ assert.strictEqual(new URL("sc:", "https://example.org/foo/bar").href, "sc:");
+ assert.strictEqual(new URL("sc:").pathname, "");
+ assert.strictEqual(new URL("blob:").href, "blob:");
+});
+
+test("a relative reference against an opaque base is rejected", (t) => {
+ assert.throws(() => new URL("test-a-colon.html", "a:"), TypeError);
+});
+
+test("a space in an opaque path is encoded only before ? or #", (t) => {
+ assert.strictEqual(
+ new URL("non-special:opaque ?hi").href,
+ "non-special:opaque %20?hi",
+ );
+ assert.strictEqual(
+ new URL("non-special:opaque #hi").href,
+ "non-special:opaque %20#hi",
+ );
+ // Not followed by ? or #, so the space stays as it is.
+ assert.strictEqual(new URL("non-special:op ok").href, "non-special:op ok");
+});
+
+test("a caret in the path is percent-encoded", (t) => {
+ assert.strictEqual(new URL("foo://host/a^b").pathname, "/a%5Eb");
+ assert.strictEqual(new URL("wss://host/a^b").pathname, "/a%5Eb");
+});
+
+test("a non-BMP character in userinfo does not eat the next one", (t) => {
+ // The emoji is a surrogate pair; the "x" after it must survive.
+ assert.strictEqual(
+ new URL("http://\u{1F600}x@host/").href,
+ "http://%F0%9F%98%80x@host/",
+ );
+ assert.strictEqual(
+ new URL("http://a:\u{1F600}x@host/").href,
+ "http://a:%F0%9F%98%80x@host/",
+ );
+});
diff --git a/packages/taler-util/src/whatwg-url.ts b/packages/taler-util/src/whatwg-url.ts
@@ -217,7 +217,13 @@ function isSpecialQueryPercentEncode(c: number) {
}
// https://url.spec.whatwg.org/#path-percent-encode-set
-const extraPathPercentEncodeSet = new Set([p("?"), p("`"), p("{"), p("}")]);
+const extraPathPercentEncodeSet = new Set([
+ p("?"),
+ p("`"),
+ p("{"),
+ p("}"),
+ p("^"),
+]);
function isPathPercentEncode(c: number) {
return isQueryPercentEncode(c) || extraPathPercentEncodeSet.has(c);
}
@@ -342,6 +348,10 @@ function isASCIIHex(c: number) {
);
}
+function isASCIIString(s: string) {
+ return !/[^\u0000-\u007F]/u.test(s);
+}
+
export class URLSearchParamsImpl {
_list: any[];
_url: any;
@@ -849,7 +859,7 @@ function parseHost(input: string, isNotSpecialArg = false) {
}
const domain = utf8DecodeWithoutBOM(percentDecodeString(input));
- const asciiDomain = domainToASCII(domain);
+ const asciiDomain = domainParser(domain);
if (asciiDomain === failure) {
return failure;
}
@@ -940,14 +950,16 @@ function serializeHost(host: number | number[] | string) {
import { punycode } from "./punycode.js";
+/**
+ * https://url.spec.whatwg.org/#concept-domain-to-ASCII
+ *
+ * Upstream uses tr46 for full IDNA processing; this copy substitutes punycode,
+ * which handles the Punycode encoding but not the IDNA mapping and validation
+ * steps around it. Domains that need those steps (non-ASCII input requiring
+ * normalization, disallowed code points, bidi and joiner rules) are therefore
+ * accepted here where the standard would reject them.
+ */
function domainToASCII(domain: string, beStrict = false) {
- // const result = tr46.toASCII(domain, {
- // checkBidi: true,
- // checkHyphens: false,
- // checkJoiners: true,
- // useSTD3ASCIIRules: beStrict,
- // verifyDNSLength: beStrict,
- // });
let result;
try {
result = punycode.toASCII(domain);
@@ -960,6 +972,25 @@ function domainToASCII(domain: string, beStrict = false) {
return result;
}
+/**
+ * https://url.spec.whatwg.org/#concept-domain-to-ASCII, the caller's half.
+ *
+ * An all-ASCII domain is lowercased and returned without going through
+ * domain-to-ASCII at all: the standard requires that for web compatibility,
+ * regardless of what ToASCII would have made of it.
+ */
+function domainParser(domain: string, beStrict = false) {
+ if (isASCIIString(domain)) {
+ return domain.toLowerCase();
+ }
+
+ const result = domainToASCII(domain, beStrict);
+ if (result === failure) {
+ return failure;
+ }
+ return result;
+}
+
function trimControlChars(url: string) {
return url.replace(/^[\u0000-\u001F\u0020]+|[\u0000-\u001F\u0020]+$/gu, "");
}
@@ -1006,7 +1037,11 @@ export interface UrlObj {
password: string;
host: string | number[] | number | null | undefined;
port: number | null;
- path: string[];
+ /**
+ * List of path segments, or a string for an opaque path (see
+ * hasAnOpaquePath).
+ */
+ path: any;
query: any;
fragment: any;
}
@@ -1065,6 +1100,7 @@ class URLStateMachine {
}
input = res;
+ this.stateOverride = stateOverride;
this.state = stateOverride || "scheme start";
this.buffer = "";
@@ -1186,7 +1222,10 @@ class URLStateMachine {
this.state = "path or authority";
++this.pointer;
} else {
- this.url.path = [""];
+ // A string rather than a list of segments: that is what marks the
+ // path as opaque, so that "sc:" serializes back as "sc:" and a
+ // relative reference against it is rejected.
+ this.url.path = "";
this.state = "opaque path";
}
} else if (!this.stateOverride) {
@@ -1328,10 +1367,13 @@ class URLStateMachine {
}
this.atFlag = true;
- // careful, this is based on buffer and has its own pointer (this.pointer != pointer) and inner chars
- const len = countSymbols(this.buffer);
- for (let pointer = 0; pointer < len; ++pointer) {
- const codePoint = this.buffer.codePointAt(pointer);
+ // careful, this iterates over the buffer's code points, independently of
+ // this.pointer. Iterating the string itself rather than indexing it
+ // keeps a surrogate pair together; indexing by code unit while counting
+ // code points would read the low surrogate on its own and drop a
+ // character.
+ for (const codePointStr of this.buffer) {
+ const codePoint = codePointStr.codePointAt(0);
if (codePoint === p(":") && !this.passwordTokenSeenFlag) {
this.passwordTokenSeenFlag = true;
@@ -1664,6 +1706,16 @@ class URLStateMachine {
} else if (c === p("#")) {
this.url.fragment = "";
this.state = "fragment";
+ } else if (c === p(" ")) {
+ this.parseError = true;
+ // Trailing spaces are the ones that would change the URL's meaning if
+ // they were dropped when it is re-parsed, so only those get encoded.
+ const remaining = this.input[this.pointer + 1];
+ if (remaining === p("?") || remaining === p("#")) {
+ this.url.path += "%20";
+ } else {
+ this.url.path += " ";
+ }
} else {
// TODO: Add: not a URL code point
if (!isNaN(c) && c !== p("%")) {
@@ -1679,7 +1731,6 @@ class URLStateMachine {
}
if (!isNaN(c)) {
- // @ts-ignore
this.url.path += utf8PercentEncodeCodePoint(
c,
isC0ControlPercentEncode,
