commit 3f0c08da0046b252f2da7e1b0891c58f5c8b04f5
parent 52eb98d4e71971d5f6b4fb82d90f0ea19d18322d
Author: Florian Dold <dold@taler.net>
Date: Sun, 13 Sep 2026 17:10:18 +0200
pogen: preserve translator comments in extracted messages
Extract translator notes from parsed token boundaries so JSX comments
survive template interpolations. Preserve multiline notes and comments
on plural forms, and avoid attaching a note to the following message.
Diffstat:
2 files changed, 132 insertions(+), 29 deletions(-)
diff --git a/packages/pogen/src/potextract.test.ts b/packages/pogen/src/potextract.test.ts
@@ -428,6 +428,41 @@ test("should not reuse a JSX-expression translator comment", () => {
);
});
+test("should extract translator comments after template interpolations", () => {
+ const out = process(`
+ <section>
+ <p>{i18n.str\`Amount \${amount}\`}</p>
+ {/* Translators: Reference for a bank transfer. */}
+ <span>{i18n.str\`Payment reference\`}</span>
+ </section>`);
+ assert.match(
+ out,
+ /#\. Translators: Reference for a bank transfer\.\n[^]*msgid "Payment reference"/,
+ );
+});
+
+test("should not treat literal translator markers as source comments", () => {
+ const out = process(`
+ <section>
+ <p>{\`\${amount} /* Translators: This is template text. */\`}</p>
+ <span>{i18n.str\`Payment reference\`}</span>
+ </section>`);
+ assert.doesNotMatch(out, /Translators:/);
+ assert.match(out, /msgid "Payment reference"/);
+});
+
+test("should not reuse an inline translator comment on the following message", () => {
+ for (const tag of ["i18n.str", 'i18n.ctx("status")']) {
+ const out = process(`{
+ label: /* Translators: Describes only the status label. */ ${tag}\`First\`,
+ detail: i18n.str\`Second\`,
+ }`);
+ assert.equal(out.match(/Translators:/g)?.length, 1);
+ assert.match(out, /msgid "First"/);
+ assert.match(out, /msgid "Second"/);
+ }
+});
+
test("should attach a translator comment between JSX attributes", () => {
const out = process(`
<SettingsToggle
@@ -452,6 +487,41 @@ test("should attach a translator comment in a ternary arm", () => {
);
});
+test("should keep all lines of a translator comment inside an expression", () => {
+ const out = process(`
+ const name = suppliedName ||
+ // Translators: Fallback when the merchant supplied neither
+ // a product name nor a description.
+ i18n.str\`Product\`;`);
+ assert.match(
+ out,
+ /#\. Translators: Fallback when the merchant supplied neither\n#\. a product name nor a description\.\n/,
+ );
+});
+
+test("should prefer an inline translator note to a preceding ordinary comment", () => {
+ const out = process(`
+ // The button starts an operation.
+ const label = /* Translators: Adds digital cash to the wallet. */ i18n.str\`Withdraw\`;`);
+ assert.match(out, /#\. Translators: Adds digital cash to the wallet\.\n/);
+ assert.doesNotMatch(out, /The button starts an operation/);
+});
+
+test("should extract translator notes on both plural forms", () => {
+ const out = process(`
+ i18n.plural(
+ count,
+ /* Translators: Token is a merchant-issued discount or pass. */ i18n.lazy\`\${count} token\`,
+ /* Translators: %1$s is the number of tokens. */ i18n.lazy\`\${count} tokens\`,
+ );`);
+ assert.match(
+ out,
+ /#\. Translators: Token is a merchant-issued discount or pass\.\n/,
+ );
+ assert.match(out, /#\. Translators: %1\$s is the number of tokens\.\n/);
+ assert.match(out, /msgid "%1\$s token"\nmsgid_plural "%1\$s tokens"/);
+});
+
//
// Duplicates.
//
diff --git a/packages/pogen/src/potextract.ts b/packages/pogen/src/potextract.ts
@@ -72,6 +72,10 @@ function getComment(
lastTokLine: number,
node: ts.Node,
): string {
+ // Explicit translator notes take precedence over other preceding comments,
+ // including when the note is inline on the same line as the message.
+ const translatorComment = getNearbyTranslatorComment(sourceFile, node);
+ if (translatorComment) return translatorComment;
let lc = ts.getLineAndCharacterOfPosition(
sourceFile,
node.getStart(sourceFile),
@@ -102,6 +106,13 @@ function getComment(
if (endLineOf(found[first]) != lc.line - 1) {
return getNearbyTranslatorComment(sourceFile, node);
}
+ if (
+ containsTranslatedMessage(
+ sourceFile.text.slice(found[first].end, node.getStart(sourceFile)),
+ )
+ ) {
+ return getNearbyTranslatorComment(sourceFile, node);
+ }
// A run of consecutive "//" lines is one comment, just like in xgettext.
if (found[first].kind === ts.SyntaxKind.SingleLineCommentTrivia) {
while (
@@ -128,11 +139,17 @@ function getComment(
.replace(/(\n[ \t]*?)?[*][/]$/, "");
break;
}
- lines.push(text);
+ lines.push(text.trimEnd());
}
return lines.join("\n");
}
+function containsTranslatedMessage(text: string): boolean {
+ return /(?:<\s*i18n\.(?:Translate|TranslateSwitch)\b|(?:\bi18n\.[A-Za-z]+(?:\([^)]*\))?|\bt)\s*`)/.test(
+ text,
+ );
+}
+
const translatorCommentCache = new WeakMap<
ts.SourceFile,
Array<{ pos: number; end: number; text: string }>
@@ -144,34 +161,52 @@ function translatorComments(
const cached = translatorCommentCache.get(sourceFile);
if (cached) return cached;
- const comments: Array<{ pos: number; end: number; text: string }> = [];
- const scanner = ts.createScanner(
- ts.ScriptTarget.Latest,
- false,
- ts.LanguageVariant.JSX,
- sourceFile.text,
- );
- for (
- let token = scanner.scan();
- token !== ts.SyntaxKind.EndOfFileToken;
- token = scanner.scan()
- ) {
- if (
- token !== ts.SyntaxKind.SingleLineCommentTrivia &&
- token !== ts.SyntaxKind.MultiLineCommentTrivia
- ) {
- continue;
+ const byPosition = new Map<number, ts.CommentRange>();
+ const collectAt = (position: number): void => {
+ const ranges = [
+ ...(ts.getLeadingCommentRanges(sourceFile.text, position) ?? []),
+ ...(ts.getTrailingCommentRanges(sourceFile.text, position) ?? []),
+ ];
+ for (const range of ranges) {
+ byPosition.set(range.pos, range);
}
- const pos = scanner.getTokenPos();
- const end = scanner.getTextPos();
- const raw = sourceFile.text.slice(pos, end);
+ };
+ // Use parsed token boundaries: a standalone scanner needs parser-directed
+ // rescanning to distinguish template tails, JSX text, and regular expressions.
+ // getChildren also visits the braces around comment-only JSX expressions.
+ const visit = (node: ts.Node): void => {
+ collectAt(node.pos);
+ collectAt(node.end);
+ for (const child of node.getChildren(sourceFile)) visit(child);
+ };
+ visit(sourceFile);
+ const ranges = [...byPosition.values()].sort((a, b) => a.pos - b.pos);
+ const comments: Array<{ pos: number; end: number; text: string }> = [];
+ for (let index = 0; index < ranges.length; index++) {
+ const range = ranges[index];
+ let end = range.end;
+ let raw = sourceFile.text.slice(range.pos, end);
if (!/\bTranslators?:/i.test(raw)) continue;
+ // A marked // comment may continue on subsequent unmarked // lines.
+ if (range.kind === ts.SyntaxKind.SingleLineCommentTrivia) {
+ while (index + 1 < ranges.length) {
+ const next = ranges[index + 1];
+ if (
+ next.kind !== ts.SyntaxKind.SingleLineCommentTrivia ||
+ !/^[ \t]*\r?\n[ \t]*$/.test(sourceFile.text.slice(end, next.pos))
+ )
+ break;
+ raw += "\n" + sourceFile.text.slice(next.pos, next.end);
+ end = next.end;
+ index++;
+ }
+ }
const text = raw
- .replace(/^[/][/]\s*/, "")
+ .replace(/^[/][/]\s*/gm, "")
.replace(/^[/][*]\s*/, "")
.replace(/\s*[*][/]$/, "")
.trim();
- comments.push({ pos, end, text });
+ comments.push({ pos: range.pos, end, text });
}
translatorCommentCache.set(sourceFile, comments);
return comments;
@@ -205,11 +240,7 @@ function getNearbyTranslatorComment(
const between = sourceFile.text.slice(comment.end, start);
// Do not reuse one comment for a later message in the same JSX block.
- if (
- /(?:<\s*i18n\.(?:Translate|TranslateSwitch)\b|(?:\bi18n\.[A-Za-z]+|\bt)\s*`)/.test(
- between,
- )
- ) {
+ if (containsTranslatedMessage(between)) {
return "";
}
return comment.text;
@@ -926,7 +957,9 @@ function processNode(
sourceFile,
searchScreenId(parents, sourceFile),
),
- comments: comment ? [comment] : [],
+ comments: [
+ ...new Set([comment, t1.comment, t2.comment].filter(Boolean)),
+ ],
refs: [sourceRef(projectPrefix, sourceFile, line)],
context: path.ctx,
msgid: content,
