commit d2addd170b64a9cebde860fa380915d44ca5eccd
parent 87b11c9e2571022cba7145732a66a76f6831a644
Author: Florian Dold <dold@taler.net>
Date: Sat, 8 Aug 2026 08:14:31 +0200
pogen: tooling fixes and improvements
Diffstat:
7 files changed, 1400 insertions(+), 322 deletions(-)
diff --git a/packages/pogen/README.md b/packages/pogen/README.md
@@ -1,79 +1,142 @@
-# pogen - string extraction for internationalizing TypeScript programs
+# pogen — string extraction for internationalizing TypeScript programs
-The ``pogen`` tool extracts internationalizable strings from TypeScript programs.
+`pogen` extracts translatable strings from TypeScript/TSX sources into a
+gettext `.pot` template, merges that template into per-language `.po` files, and
+emits a `strings.ts` catalogue the application imports at runtime.
+## Invocation
-## Invocation and Configuration
+Run from the root of an NPM package. The input files are whatever the TypeScript
+compiler would use, taken from the package's `tsconfig.json`.
-The ``pogen`` tool must be called from the root of an NPM package.
+```shell
+pogen extract # sources -> src/i18n/<domain>.pot
+pogen merge # <domain>.pot -> src/i18n/*.po (runs GNU msgmerge)
+pogen emit # src/i18n/*.po -> src/i18n/strings.ts
+pogen check # validate the catalogues; exits non-zero on any problem
+```
-The input files are determined from the ``tsconfig.json`` file at the root of
-the package. All input files inside the package that the compiler would use
-are automatically processed.
+`merge` requires **GNU gettext** (`msgmerge`) on the `PATH`; `check` uses
+`msgfmt` when it is available and skips that half with a message when it is not.
-Further configuration options are specified in the package's ``package.json`` file.
-The following configuration options are supported:
+## Configuration
-```
+One key, in the package's `package.json`:
+
+```json
{
+ "pogen": {
+ "domain": "taler-merchant-webui-ng"
+ }
+}
+```
- // [ ... ]
+`domain` names the template: `src/i18n/<domain>.pot`. Paths are otherwise fixed —
+`src/i18n/*.po` in, `src/i18n/strings.ts` out. Two optional files are read if
+present: `src/i18n/poheader` (replaces the default `.pot` header) and
+`src/i18n/strings-prelude` (replaces the preamble of the emitted `strings.ts`).
- "pogen": {
- // Output location of the pofile (mandatory)
- "pofile": "...",
+> Earlier versions of this document described `pofile`, `plainI18nPackage` and
+> `reactI18nPackage`, and claimed extraction was scoped to strings imported from
+> a named package. None of that was ever implemented. Extraction is purely
+> syntactic — see below — and `domain` is the only key read.
- // Calls to plain i18n functions are extracted if they
- // are imported from this package.
- "plainI18nPackage": "@gnu-taler/taler-util",
+## What gets extracted
- // Calls to react-style i18n functions are extracted if they
- // are imported from this package.
- "reactI18nPackage": "@gnu-taler/preact-i18n",
- }
+Extraction matches **tagged templates** whose tag begins with the identifier
+`t` or `i18n`, and the `i18n` JSX components. It is syntactic: any member of
+those roots works (`i18n.str`, `i18n.lazy`, `t.context(...)`), and nothing checks
+where the identifier came from.
-}
+```ts
+t`Hello world`
+t`Hello ${user}` // msgid "Hello %1$s"
+i18n.str`Hello world`
+i18n.context("navigation")`Orders` // msgctxt "navigation"
```
+```tsx
+<i18n.Translate>Hello, world</i18n.Translate>
-## Syntax
+// Placeholders must be wrapped in an element, not bare:
+<i18n.Translate>Hello, <span>{userName}</span></i18n.Translate>
-Two flavors of syntax are supported:
+<i18n.TranslateSwitch target={n}>
+ <i18n.TranslateSingular>I have <span>{n}</span> apple</i18n.TranslateSingular>
+ <i18n.TranslatePlural>I have <span>{n}</span> apples</i18n.TranslatePlural>
+</i18n.TranslateSwitch>
+```
-Template strings:
+Plurals via a call take the two forms below; both work.
+```ts
+i18n.plural(i18n.lazy`one apple`, i18n.lazy`${n} apples`)
+i18n.plural(n, i18n.lazy`one apple`, i18n.lazy`${n} apples`)
```
-import { i18n } from "@gnu-taler/taler-util";
-console.log(i18n.str`Hello World`);
-console.log(i18n.str`Hello ${user}`);
+### What does *not* get extracted
+
+**A plain call is invisible to the extractor**, even though a runtime `t` may
+accept one:
-console.log(i18n.plural(n, i18n.lazy`I have ${n} apple`, i18n.lazy`I have ${n} apples`));
+```ts
+t("Charge amount") // NOT extracted — pogen warns
+t(someVariable) // NOT extracted — cannot be, in general
+t(`All Products (${n})`) // NOT extracted — pogen warns; a new msgid per value
```
-React components:
+`pogen extract` reports the first and third with a file and line. The second is
+silent, because it is indistinguishable from legitimate use.
-```
+To translate a string that lives in a data table, do not translate it at the
+call site — build the table from a factory that takes `t` as a parameter **named
+`t`**, so the tagged templates inside it are extracted normally:
-import {
- Translate,
- TranslateSwitch,
- TranslateSingular,
- TranslatePlural
-} from "@gnu-taler/preact-i18n";
+```ts
+function tabLabels(t: TranslateFn) {
+ return { all: t`All`, paid: t`Paid` }; // extracted
+}
+```
-<Translate>Hello, World</Translate>
+### Placeholders and `%`
-// Placeholders are other React elements
-<Translate>Hello, <span className="highlight">{userName}<span></Translate>
+An interpolation becomes `%1$s`, `%2$s`, … in source order. A literal `%` is
+passed through untouched and needs no escaping: these strings are **not** C
+format strings — the runtimes substitute `%N$s` by string replacement — and
+`pogen` deliberately does not emit a `#, c-format` flag. Placeholder consistency
+between a msgid and its translations is checked by `pogen check` instead.
-// Plain placeholders are not supported, they must be surrounded
-// by an element:
-// WRONG: <Translate>Hello, {userName}</Translate>
+### Comments
-<TranslateSwitch n={numApples}>
- <TranslateSingular>I have <span>{n}</span> apple</TranslateSingular>
- <TranslatePlural>I have <span>{n}</span> apple</TranslatePlural>
-</TranslateSwitch>
+A comment on the line immediately above a translatable string becomes a `#.`
+comment for the translator. A run of `//` lines is kept whole; a blank line
+between the comment and the string breaks the association.
+```ts
+// Shown on the receipt, so keep it short.
+t`Thank you`
```
+
+Note `msgmerge` regenerates the `#.` block from the `.pot` on every merge, so a
+`#.` comment written by hand in a `.po` file will not survive. A plain `# `
+translator comment does survive.
+
+## What `pogen check` verifies
+
+Exits non-zero on any of: `msgfmt --check-format` failures; a translation whose
+`%N$s` placeholders do not match its msgid; or a non-`en` catalogue whose
+completeness falls below the threshold the language picker uses to auto-select a
+language. Run it in CI — the other three subcommands are deliberately permissive
+and will not tell you a catalogue is broken.
+
+## Notes and known limitations
+
+- **Extraction is not scoped to a package.** `pogen` walks every source file the
+ TypeScript program reaches, which in a workspace includes sibling packages. A
+ package's `.pot` may therefore contain strings that belong to a dependency.
+- **`completeness`** in `strings.ts` is `translated / (translated + fuzzy +
+ untranslated)`. It measures coverage, not quality: a msgstr that merely repeats
+ the English counts as translated.
+- **`en` is special-cased to 100.** The `en.po` files are English-to-English
+ identity catalogues that exist so `en` appears in the language list; they are
+ not filled in.
diff --git a/packages/pogen/src/check.ts b/packages/pogen/src/check.ts
@@ -0,0 +1,191 @@
+/*
+ This file is part of GNU Taler
+ (C) 2026 Taler Systems S.A.
+
+ GNU Taler is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+ */
+
+/**
+ * Validate the `.po` catalogues of a package.
+ *
+ * Three checks, each of which can fail the run on its own:
+ *
+ * - `msgfmt --check-format`, if GNU gettext is installed;
+ * - a placeholder comparison between msgid and msgstr, which needs nothing
+ * but this package and so always runs;
+ * - the coverage threshold that decides whether a browser will auto-select
+ * the language at all.
+ */
+
+import * as child_process from "node:child_process";
+import * as fs from "node:fs";
+import * as gettextParser from "gettext-parser";
+import * as glob from "glob";
+import { poToStrings } from "./po2ts.js";
+
+/**
+ * Below this, `web-util`'s `useLang` will not auto-select the language and the
+ * browser silently falls back to English — see
+ * `packages/web-util/src/hooks/useLang.ts` (`MIN_LANG_COVERAGE_THRESHOLD`).
+ * Failing here means the regeneration commit that would cause that is loud.
+ */
+export const MIN_LANG_COVERAGE_THRESHOLD = 85;
+
+/**
+ * The placeholders a message uses, sorted so two messages can be compared
+ * regardless of the word order the target language needs.
+ *
+ * Kept identical to the check in
+ * `packages/merchant-webui-ng/src/i18n/catalog.test.ts`.
+ */
+export function placeholders(s: string): string[] {
+ return (s.match(/%\d+\$s|%s|%%/g) ?? []).sort();
+}
+
+/**
+ * Compare the placeholders of every live entry of one catalogue.
+ *
+ * "Live" means it will actually be emitted into `strings.ts`: it has a
+ * translation and is not marked fuzzy. A msgstr carrying a `%1$s` that its
+ * msgid does not have renders the placeholder literally to the user.
+ *
+ * Returns one human-readable line per problem; an empty array means clean.
+ */
+export function checkPlaceholders(poText: string, name: string): string[] {
+ const parsed = gettextParser.po.parse(poText);
+ const problems: string[] = [];
+
+ for (const msgctxt of Object.keys(parsed.translations)) {
+ const bucket = parsed.translations[msgctxt] || {};
+ for (const msgid of Object.keys(bucket)) {
+ if (msgid === "") {
+ continue;
+ }
+ const entry = bucket[msgid];
+ const flags = (entry.comments && entry.comments.flag) || "";
+ if (flags.split(",").some((f) => f.trim() === "fuzzy")) {
+ continue;
+ }
+ const label = msgctxt === "" ? msgid : `${msgctxt}|${msgid}`;
+ const msgstr = entry.msgstr || [];
+ for (let i = 0; i < msgstr.length; i++) {
+ if (!msgstr[i]) {
+ // Untranslated: falls back to English, nothing to compare.
+ continue;
+ }
+ // For a plural, form 0 answers the singular msgid and every later
+ // form answers the msgid_plural.
+ const source =
+ entry.msgid_plural && i > 0 ? entry.msgid_plural : msgid;
+ const a = placeholders(source);
+ const b = placeholders(msgstr[i]);
+ if (a.join() !== b.join()) {
+ problems.push(
+ `${name}: placeholder mismatch ${JSON.stringify(
+ source,
+ )} -> ${JSON.stringify(msgstr[i])}`,
+ );
+ }
+ }
+ }
+ }
+ return problems;
+}
+
+function haveGettext(): boolean {
+ try {
+ child_process.execFileSync("msgfmt", ["--version"], { stdio: "ignore" });
+ return true;
+ } catch (e) {
+ return false;
+ }
+}
+
+/**
+ * Run every check over `src/i18n/*.po`, relative to the current directory.
+ * Exits non-zero if anything at all is wrong.
+ */
+export function check(): void {
+ const files = glob.sync("src/i18n/*.po");
+
+ if (files.length === 0) {
+ console.error("no .po files found in src/i18n/");
+ process.exit(1);
+ }
+
+ let failed = false;
+
+ if (!haveGettext()) {
+ console.log(
+ "skipping 'msgfmt --check-format': GNU gettext is not installed " +
+ "(install gettext to enable it); the placeholder and coverage " +
+ "checks below do not need it",
+ );
+ } else {
+ for (const f of files) {
+ const r = child_process.spawnSync(
+ "msgfmt",
+ ["--check-format", "--output-file=/dev/null", f],
+ { encoding: "utf-8" },
+ );
+ const output = `${r.stdout || ""}${r.stderr || ""}`.trim();
+ if (output) {
+ console.error(output);
+ }
+ if (r.status !== 0) {
+ console.error(`msgfmt --check-format failed on ${f}`);
+ failed = true;
+ }
+ }
+ }
+
+ for (const f of files) {
+ const poText = fs.readFileSync(f, "utf-8");
+
+ const problems = checkPlaceholders(poText, f);
+ for (const p of problems) {
+ console.error(p);
+ }
+ if (problems.length > 0) {
+ failed = true;
+ }
+
+ // The same number `emit` will write into strings.ts.
+ let completeness: number;
+ let lang: string;
+ try {
+ const m = f.match(/([a-zA-Z0-9-_]+)\.po$/);
+ const strings = poToStrings(poText, m ? m[1] : f);
+ completeness = strings.completeness;
+ lang = strings.lang;
+ } catch (e) {
+ console.error(`${f}: ${e instanceof Error ? e.message : e}`);
+ failed = true;
+ continue;
+ }
+ if (lang !== "en" && completeness < MIN_LANG_COVERAGE_THRESHOLD) {
+ console.error(
+ `${f}: only ${completeness}% translated, below the ` +
+ `${MIN_LANG_COVERAGE_THRESHOLD}% threshold at which a browser stops ` +
+ `auto-selecting '${lang}' and silently falls back to English`,
+ );
+ failed = true;
+ } else {
+ console.log(`${f}: ${completeness}% translated`);
+ }
+ }
+
+ if (failed) {
+ process.exit(1);
+ }
+ console.log("all catalogues pass");
+}
diff --git a/packages/pogen/src/po2ts.test.ts b/packages/pogen/src/po2ts.test.ts
@@ -0,0 +1,196 @@
+/*
+ This file is part of GNU Taler
+ (C) 2026 Taler Systems S.A.
+
+ GNU Taler is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+ */
+
+import { test } from "node:test";
+import assert from "node:assert";
+import { CONTEXT_DELIMITER, poToStrings } from "./po2ts.js";
+
+function header(lang: string, pluralForms = "nplurals=2; plural=(n != 1);"): string {
+ return `msgid ""
+msgstr ""
+"Content-Type: text/plain; charset=UTF-8\\n"
+"Content-Transfer-Encoding: 8bit\\n"
+"Language: ${lang}\\n"
+"Plural-Forms: ${pluralForms}\\n"
+
+`;
+}
+
+test("a msgctxt entry is emitted, keyed the way jed looks it up", () => {
+ const s = poToStrings(
+ header("de") +
+ `msgctxt "title"
+msgid "%1$s: Settings"
+msgstr "%1$s: Einstellungen"
+`,
+ "de",
+ );
+ const messages = s.locale_data.messages;
+ assert.deepEqual(
+ messages[`title${CONTEXT_DELIMITER}%1$s: Settings`],
+ ["%1$s: Einstellungen"],
+ "context entry must be keyed msgctxt + EOT + msgid",
+ );
+ // jed's Jed.context_delimiter, EOT (U+0004).
+ assert.equal(CONTEXT_DELIMITER.length, 1);
+ assert.equal(CONTEXT_DELIMITER.charCodeAt(0), 4);
+ // The bare msgid must not be claimed: `i18n.str` and `i18n.ctx("title")`
+ // are different messages.
+ assert.equal(messages["%1$s: Settings"], undefined);
+ assert.equal(s.completeness, 100);
+});
+
+test("a msgid with and without context are separate entries", () => {
+ const s = poToStrings(
+ header("de") +
+ `msgid "Order"
+msgstr "Bestellung"
+
+msgctxt "verb"
+msgid "Order"
+msgstr "Bestellen"
+`,
+ "de",
+ );
+ const messages = s.locale_data.messages;
+ assert.deepEqual(messages["Order"], ["Bestellung"]);
+ assert.deepEqual(messages[`verb${CONTEXT_DELIMITER}Order`], ["Bestellen"]);
+});
+
+test("a fuzzy entry is dropped from the catalogue but counts as untranslated", () => {
+ const s = poToStrings(
+ header("de") +
+ `msgid "Hello"
+msgstr "Hallo"
+
+#, fuzzy
+msgid "Goodbye"
+msgstr "Auf Wiedersehen"
+`,
+ "de",
+ );
+ assert.deepEqual(s.locale_data.messages["Hello"], ["Hallo"]);
+ assert.equal(
+ s.locale_data.messages["Goodbye"],
+ undefined,
+ "a fuzzy entry must not be emitted",
+ );
+ // 1 of 2, not 1 of 1: the fuzzy entry renders English, so it is missing.
+ assert.equal(s.completeness, 50);
+});
+
+test("an empty msgstr counts in the denominator only", () => {
+ const s = poToStrings(
+ header("de") +
+ `msgid "Hello"
+msgstr "Hallo"
+
+msgid "Goodbye"
+msgstr ""
+
+msgid "Thanks"
+msgstr ""
+`,
+ "de",
+ );
+ assert.deepEqual(s.locale_data.messages["Goodbye"], [""]);
+ assert.equal(s.completeness, 33);
+});
+
+test("a plural keeps [msgid_plural, ...msgstr] and needs every form filled in", () => {
+ const s = poToStrings(
+ header("de") +
+ `msgid "one file"
+msgid_plural "%1$s files"
+msgstr[0] "eine Datei"
+msgstr[1] "%1$s Dateien"
+`,
+ "de",
+ );
+ assert.deepEqual(s.locale_data.messages["one file"], [
+ "%1$s files",
+ "eine Datei",
+ "%1$s Dateien",
+ ]);
+ assert.equal(s.completeness, 100);
+});
+
+test("a plural with an unfilled form does not count as translated", () => {
+ // The old counter looked at `v[0]`, which for a plural is the English
+ // msgid_plural, so every plural counted as done no matter what.
+ const s = poToStrings(
+ header("de") +
+ `msgid "one file"
+msgid_plural "%1$s files"
+msgstr[0] "eine Datei"
+msgstr[1] ""
+`,
+ "de",
+ );
+ assert.deepEqual(s.locale_data.messages["one file"], [
+ "%1$s files",
+ "eine Datei",
+ "",
+ ]);
+ assert.equal(s.completeness, 0);
+});
+
+test("an entirely untranslated 'en' catalogue is still 100% complete", () => {
+ // The en.po files in this tree are English-to-English identity catalogues
+ // with every msgstr empty. Reporting their honest 0% would drop 'en' out of
+ // the language picker.
+ const s = poToStrings(
+ header("en") +
+ `msgid "Hello"
+msgstr ""
+
+msgid "Goodbye"
+msgstr ""
+`,
+ "en",
+ );
+ assert.equal(s.lang, "en");
+ assert.equal(s.completeness, 100);
+});
+
+test("an empty catalogue reports 0 rather than NaN", () => {
+ const s = poToStrings(header("de"), "de");
+ assert.equal(s.completeness, 0);
+});
+
+test("a catalogue without a Language header is rejected", () => {
+ assert.throws(
+ () =>
+ poToStrings(
+ `msgid ""
+msgstr ""
+"Content-Type: text/plain; charset=UTF-8\\n"
+
+msgid "Hello"
+msgstr "Hallo"
+`,
+ "de",
+ ),
+ /Language/,
+ );
+});
+
+test("the header carries lang and plural forms through", () => {
+ const s = poToStrings(header("fr", "nplurals=2; plural=(n > 1);"), "fr");
+ assert.equal(s.lang, "fr");
+ assert.equal(s.plural_forms, "nplurals=2; plural=(n > 1);");
+ assert.equal(s.domain, "messages");
+});
diff --git a/packages/pogen/src/po2ts.ts b/packages/pogen/src/po2ts.ts
@@ -39,7 +39,7 @@ interface pojsonType {
}
// ----------- end pf po2json
-interface StringsType {
+export interface StringsType {
// X-Domain or 'messages'
domain: string;
lang: string;
@@ -65,6 +65,121 @@ export interface StringsType {
const DEFAULT_STRING_PRELUDE = `${TYPES_FOR_STRING_PRELUDE}export const strings: Record<string,StringsType> = {};\n\n`
+/**
+ * Separator between a msgctxt and its msgid in the emitted catalogue.
+ *
+ * This is jed's `Jed.context_delimiter` (EOT, U+0004) — see
+ * `jed/jed.js:117,263` — and jed is what actually reads the emitted
+ * `strings.ts` at runtime, via `setupI18n` and `i18n.ctx(...)` in
+ * `@gnu-taler/taler-util` (`src/i18n.ts:66-76`, which calls
+ * `jed.translate(...).withContext(ctx)`). Keying context entries any other
+ * way puts them in the file but leaves them unreachable.
+ */
+export const CONTEXT_DELIMITER = "\u0004";
+
+/**
+ * Convert the text of one `.po` file into the value that is emitted as
+ * `strings['<lang>']`.
+ *
+ * Pure: no file system access, no `process.exit`. `catalogueName` is only
+ * used in error messages (it is the `<lang>` part of the file name); the
+ * `lang` of the result comes from the `Language:` header of the catalogue.
+ */
+export function poToStrings(poText: string, catalogueName: string): StringsType {
+ const parsedPo = gettextParser.po.parse(poText);
+ const messages: any = {
+ "": {
+ domain: "messages",
+ lang: parsedPo.headers["Language"] || parsedPo.headers["language"] || "",
+ plural_forms:
+ parsedPo.headers["Plural-Forms"] ||
+ parsedPo.headers["plural-forms"] ||
+ "",
+ },
+ };
+
+ // Every message the catalogue is responsible for, and the subset of those
+ // that actually carry a usable translation. Counted here rather than over
+ // `messages` below, because entries that are dropped from the emitted
+ // catalogue (fuzzy ones) still have to count against the total: a fuzzy
+ // entry renders as English, so it is untranslated as far as the reader is
+ // concerned.
+ let totalKeys = 0;
+ let totalTranslated = 0;
+
+ // `gettext-parser` files each entry under its msgctxt, with the empty string
+ // for entries that have none. Every bucket has to be read: only looking at
+ // `translations[""]` silently discards every string extracted through
+ // `i18n.ctx(...)`.
+ for (const msgctxt of Object.keys(parsedPo.translations)) {
+ const contextTranslations = parsedPo.translations[msgctxt] || {};
+ for (const msgid of Object.keys(contextTranslations)) {
+ if (msgid === "") {
+ // The catalogue header, already handled above.
+ continue;
+ }
+ const entry = contextTranslations[msgid];
+ const key =
+ msgctxt === "" ? msgid : `${msgctxt}${CONTEXT_DELIMITER}${msgid}`;
+ totalKeys++;
+
+ // Treat fuzzy entries as untranslated (standard gettext behaviour):
+ // msgmerge seeds fuzzy translations from unrelated nearby strings, so
+ // shipping them would surface wrong-meaning text. Fall back to English.
+ const flags = (entry.comments && entry.comments.flag) || "";
+ if (flags.split(",").some((f) => f.trim() === "fuzzy")) {
+ continue;
+ }
+
+ const msgstr = entry.msgstr || [];
+ if (entry.msgid_plural) {
+ // A plural is only translated once *every* form is filled in; a
+ // catalogue with `msgstr[1] ""` renders English for those counts.
+ if (msgstr.length > 0 && msgstr.every((s) => !!s)) {
+ totalTranslated++;
+ }
+ messages[key] = [entry.msgid_plural, ...msgstr];
+ } else {
+ if (msgstr.length > 0 && !!msgstr[0]) {
+ totalTranslated++;
+ }
+ messages[key] = msgstr;
+ }
+ }
+ }
+
+ const poAsJson: pojsonType = {
+ domain: "messages",
+ locale_data: {
+ messages: messages as MessagesType,
+ },
+ };
+ const header = poAsJson.locale_data.messages[""];
+ if (!header.lang) {
+ throw new Error(
+ `missing 'Language' property in the catalogue '${catalogueName}'`,
+ );
+ }
+
+ // 'en' is always complete. Keep this: the `en.po` files in this tree are
+ // English-to-English identity catalogues with every msgstr empty, so they
+ // honestly measure 0%. Reporting that would push `en` below the coverage
+ // threshold in `web-util/src/hooks/useLang.ts` and drop it out of the
+ // language picker (`web-util/src/context/translation.ts:85-90`), which
+ // filters on a non-zero completeness.
+ const completeness =
+ header.lang === "en"
+ ? 100
+ : Math.floor((totalTranslated * 100) / totalKeys);
+
+ return {
+ locale_data: poAsJson.locale_data,
+ domain: poAsJson.domain,
+ plural_forms: header.plural_forms,
+ lang: header.lang,
+ completeness: Number.isNaN(completeness) ? 0 : completeness,
+ };
+}
export function po2ts(): void {
const files = glob.sync("src/i18n/*.po");
@@ -94,61 +209,14 @@ export function po2ts(): void {
}
const lang = m[1];
- const poBuffer = fs.readFileSync(filename);
- const parsedPo = gettextParser.po.parse(poBuffer);
- const messages: any = {
- "": {
- domain: "messages",
- lang: parsedPo.headers["Language"] || parsedPo.headers["language"] || "",
- plural_forms:
- parsedPo.headers["Plural-Forms"] ||
- parsedPo.headers["plural-forms"] ||
- "",
- },
- };
- const domainTranslations = parsedPo.translations[""] || {};
- for (const msgid of Object.keys(domainTranslations)) {
- if (msgid === "") {
- continue;
- }
- const entry = domainTranslations[msgid];
- // Treat fuzzy entries as untranslated (standard gettext behaviour):
- // msgmerge seeds fuzzy translations from unrelated nearby strings, so
- // shipping them would surface wrong-meaning text. Fall back to English.
- const flags = (entry.comments && entry.comments.flag) || "";
- if (flags.split(",").some((f) => f.trim() === "fuzzy")) {
- continue;
- }
- if (entry.msgid_plural) {
- messages[msgid] = [entry.msgid_plural, ...entry.msgstr];
- } else {
- messages[msgid] = entry.msgstr;
- }
- }
- const poAsJson: pojsonType = {
- domain: "messages",
- locale_data: {
- messages: messages as MessagesType,
- },
- };
- const header = poAsJson.locale_data.messages[""];
- if (!header.lang) {
- console.error("error: missing 'Language' property in " + filename);
+ const poText = fs.readFileSync(filename, "utf-8");
+ let strings: StringsType;
+ try {
+ strings = poToStrings(poText, lang);
+ } catch (e) {
+ console.error(`error: ${e instanceof Error ? e.message : e} (${filename})`);
process.exit(1);
}
- const total = calculateTotalTranslations(poAsJson.locale_data.messages)
- const completeness =
- (header.lang === "en"
- ? 100 // 'en' is always complete
- : Math.floor(total.translations * 100 / total.keys));
-
- const strings: StringsType = {
- locale_data: poAsJson.locale_data,
- domain: poAsJson.domain,
- plural_forms: header.plural_forms,
- lang: header.lang,
- completeness: Number.isNaN(completeness) ? 0 : completeness,
- }
const value = JSON.stringify(strings, undefined, 2)
const s = `strings['${lang}'] = ${value};\n\n`
chunks.push(s);
@@ -158,25 +226,3 @@ export function po2ts(): void {
fs.writeFileSync("src/i18n/strings.ts", tsContents);
}
-
-function calculateTotalTranslations(msgs: MessagesType): { keys: number, translations: number } {
- const kv = Object.entries(msgs)
- const [keys, translations] = kv.reduce(([total, withTranslation], translation) => {
- if (!translation || translation.length !== 2 || !translation[1]) {
- //current key is empty
- return [total, withTranslation]
- }
- const v = translation[1]
- if (!Array.isArray(v)) {
- // this is not a translation
- return [total, withTranslation]
- }
- if (!v.length || !v[0].length) {
- //translation is missing
- return [total + 1, withTranslation]
- }
- //current key has a translation
- return [total + 1, withTranslation + 1]
- }, [0, 0])
- return { keys, translations }
-}
-\ No newline at end of file
diff --git a/packages/pogen/src/pogen.ts b/packages/pogen/src/pogen.ts
@@ -3,12 +3,30 @@ import * as child_process from "child_process";
import * as fs from "node:fs";
import * as glob from "glob";
import { po2ts } from "./po2ts.js";
+import { check } from "./check.js";
function usage(): never {
- console.log("usage: pogen <extract|merge|emit>");
+ console.log("usage: pogen <extract|merge|emit|check>");
process.exit(1);
}
+/**
+ * `merge` shells out to GNU gettext. Without this the absence of the toolchain
+ * surfaces as a raw execSync throw with a shell's "command not found" buried in
+ * it.
+ */
+function requireGettextTool(tool: string): void {
+ try {
+ child_process.execFileSync(tool, ["--version"], { stdio: "ignore" });
+ } catch (e) {
+ console.error(
+ `'${tool}' not found: this subcommand needs GNU gettext installed ` +
+ `(Debian/Ubuntu: 'apt install gettext')`,
+ );
+ process.exit(1);
+ }
+}
+
export function main() {
const subcommand = process.argv[2];
if (process.argv.includes("--help") || !subcommand) {
@@ -30,12 +48,16 @@ export function main() {
console.error("missing 'pogen.domain' field in package.json");
process.exit(1);
}
+ requireGettextTool("msgmerge");
const files = glob.sync("src/i18n/*.po");
console.log(files);
for (const f of files) {
console.log(`merging ${f}`);
+ // --previous keeps the old msgid on every entry msgmerge marks fuzzy,
+ // as a '#| msgid' line. Without it a translator sees a fuzzy string
+ // with no way to tell what it was matched against.
child_process.execSync(
- `msgmerge -o '${f}' '${f}' 'src/i18n/${poDomain}.pot'`,
+ `msgmerge --previous -o '${f}' '${f}' 'src/i18n/${poDomain}.pot'`,
);
}
break;
@@ -43,6 +65,9 @@ export function main() {
case "emit":
po2ts();
break;
+ case "check":
+ check();
+ break;
default:
console.error(`unknown subcommand '${subcommand}'`);
usage();
diff --git a/packages/pogen/src/potextract.test.ts b/packages/pogen/src/potextract.test.ts
@@ -17,7 +17,7 @@
import { test } from "node:test";
import assert from "node:assert";
import * as ts from "typescript";
-import { processFileForTesting } from "./potextract.js";
+import { ParseError, processFileForTesting } from "./potextract.js";
function wrapIntoFunction(src: string): string {
return `
@@ -28,13 +28,41 @@ return ${src}
`;
}
-function process(src: string): string {
- const source = ts.createSourceFile(
- "test.tsx",
+function sourceOf(name: string, src: string): ts.SourceFile {
+ return ts.createSourceFile(
+ name,
wrapIntoFunction(src),
ts.ScriptTarget.ES2023,
);
- return processFileForTesting(source).trim();
+}
+
+function process(src: string): string {
+ return processFileForTesting(sourceOf("test.tsx", src)).trim();
+}
+
+/**
+ * Same as process(), but over several files, so that the accumulation of
+ * source references for one msgid can be observed.
+ */
+function processMany(files: [string, string][]): string {
+ return processFileForTesting(
+ ...files.map(([name, src]) => sourceOf(name, src)),
+ ).trim();
+}
+
+/** Run f() while collecting everything it writes to stderr. */
+function captureStderr(f: () => void): string[] {
+ const original = console.error;
+ const collected: string[] = [];
+ console.error = (...args: any[]) => {
+ collected.push(args.map((a) => String(a)).join(" "));
+ };
+ try {
+ f();
+ } finally {
+ console.error = original;
+ }
+ return collected;
}
test("should extract the key from inner body", (t) => {
@@ -42,7 +70,6 @@ test("should extract the key from inner body", (t) => {
process(`<i18n.Translate>something</i18n.Translate>`),
`#. screenid: 5
#: test.tsx:4
-#, c-format
msgid "something"
msgstr ""`,
);
@@ -57,7 +84,6 @@ test("should support context on tags", (t) => {
),
`#. screenid: 5
#: test.tsx:5
-#, c-format
msgctxt "some_context"
msgid "something"
msgstr ""`,
@@ -69,7 +95,6 @@ test("should support context on string template", (t) => {
process(`return i18n.context("wire transfer")\`send\`;`),
`#. screenid: 5
#: test.tsx:4
-#, c-format
msgctxt "wire transfer"
msgid "send"
msgstr ""`,
@@ -83,14 +108,12 @@ test("should support same message id with different context", (t) => {
),
`#. screenid: 5
#: test.tsx:4
-#, c-format
msgctxt "wire transfer"
msgid "send"
msgstr ""
#. screenid: 5
#: test.tsx:4
-#, c-format
msgctxt "gift"
msgid "send"
msgstr ""`,
@@ -105,7 +128,6 @@ test("should support on string template", (t) => {
`#. screenid: 5
#. comment of the translation
#: test.tsx:6
-#, c-format
msgid "another key"
msgstr ""`,
);
@@ -123,13 +145,11 @@ test("should override screen id", (t) => {
`#. screenid: 6
#. comment of the translation
#: test.tsx:8
-#, c-format
msgid "another key"
msgstr ""`,
);
});
-
test("should support nested tags", (t) => {
assert.deepStrictEqual(
process(`
@@ -143,14 +163,366 @@ test("should support nested tags", (t) => {
`),
`#. screenid: 5
#: test.tsx:6
-#, c-format
msgid "Purging an instance %1$s"
msgstr ""
#. screenid: 5
#: test.tsx:9
-#, c-format
msgid "This cannot be undone!"
msgstr ""`,
);
});
+
+//
+// The "t" tag, which is the form used throughout merchant-webui-ng and
+// which the suite never covered.
+//
+
+test("should extract the bare t tag", (t) => {
+ assert.deepStrictEqual(
+ process(`t\`Orders\`;`),
+ `#. screenid: 5
+#: test.tsx:4
+msgid "Orders"
+msgstr ""`,
+ );
+});
+
+test("should extract the t tag with interpolation", (t) => {
+ assert.deepStrictEqual(
+ process(`t\`found \${n} orders\`;`),
+ `#. screenid: 5
+#: test.tsx:4
+msgid "found %1$s orders"
+msgstr ""`,
+ );
+});
+
+test("should extract a t tag with a context", (t) => {
+ assert.deepStrictEqual(
+ process(`t.context("navigation")\`Orders\`;`),
+ `#. screenid: 5
+#: test.tsx:4
+msgctxt "navigation"
+msgid "Orders"
+msgstr ""`,
+ );
+});
+
+//
+// Plain calls. None of these is extractable; the two literal forms must at
+// least be reported so that the gap is visible.
+//
+
+test("t(literal) is not extracted but warns", (t) => {
+ let out = "";
+ const errors = captureStderr(() => {
+ out = process(`t("Wire transfer");`);
+ });
+ assert.deepStrictEqual(out, "");
+ assert.deepStrictEqual(errors.length, 1);
+ assert.ok(
+ errors[0].includes("test.tsx:4"),
+ `expected a file:line in ${JSON.stringify(errors[0])}`,
+ );
+ assert.ok(errors[0].includes("not extractable"), errors[0]);
+});
+
+test("t(variable) is not extracted and does not warn", (t) => {
+ let out = "";
+ const errors = captureStderr(() => {
+ out = process(`t(someLabel);`);
+ });
+ assert.deepStrictEqual(out, "");
+ assert.deepStrictEqual(errors, []);
+});
+
+test("t(template with interpolation) is not extracted but warns", (t) => {
+ let out = "";
+ const errors = captureStderr(() => {
+ out = process(`t(\`All Products (\${n})\`);`);
+ });
+ assert.deepStrictEqual(out, "");
+ assert.deepStrictEqual(errors.length, 1);
+ assert.ok(errors[0].includes("test.tsx:4"), errors[0]);
+ assert.ok(errors[0].includes("msgid"), errors[0]);
+});
+
+test("a context specifier call is not mistaken for a message", (t) => {
+ let out = "";
+ const errors = captureStderr(() => {
+ out = process(`i18n.context("wire transfer")\`send\`;`);
+ });
+ assert.deepStrictEqual(
+ out,
+ `#. screenid: 5
+#: test.tsx:4
+msgctxt "wire transfer"
+msgid "send"
+msgstr ""`,
+ );
+ assert.deepStrictEqual(errors, []);
+});
+
+//
+// "%" must be passed through verbatim, because the runtime builds its lookup
+// key from the raw template parts.
+//
+
+test("should not escape % in the head of a template", (t) => {
+ assert.deepStrictEqual(
+ process(`t\`100% of \${n} done\`;`),
+ `#. screenid: 5
+#: test.tsx:4
+msgid "100% of %1$s done"
+msgstr ""`,
+ );
+});
+
+test("should not escape % in a tail fragment of a template", (t) => {
+ assert.deepStrictEqual(
+ process(`t\`\${n} is 100% done\`;`),
+ `#. screenid: 5
+#: test.tsx:4
+msgid "%1$s is 100% done"
+msgstr ""`,
+ );
+});
+
+test("should not escape % in a template without interpolation", (t) => {
+ assert.deepStrictEqual(
+ process(`t\`100% done\`;`),
+ `#. screenid: 5
+#: test.tsx:4
+msgid "100% done"
+msgstr ""`,
+ );
+});
+
+test("should never emit the c-format flag", (t) => {
+ assert.ok(!process(`t\`\${n} is 100% done\`;`).includes("c-format"));
+ assert.ok(
+ !process(`<i18n.Translate>50% off</i18n.Translate>`).includes("c-format"),
+ );
+});
+
+//
+// Comments.
+//
+
+test("should keep every line of a multi-line // comment", (t) => {
+ assert.deepStrictEqual(
+ process(`
+ // The header of the event list,
+ // so the title says what happened.
+ return i18n.str\`another key\`;`),
+ `#. screenid: 5
+#. The header of the event list,
+#. so the title says what happened.
+#: test.tsx:7
+msgid "another key"
+msgstr ""`,
+ );
+});
+
+test("should not absorb a comment separated by a blank line", (t) => {
+ assert.deepStrictEqual(
+ process(`
+ // an unrelated remark
+
+ // the actual comment
+ return i18n.str\`another key\`;`),
+ `#. screenid: 5
+#. the actual comment
+#: test.tsx:8
+msgid "another key"
+msgstr ""`,
+ );
+});
+
+test("should attach a comment to a JSX message", (t) => {
+ assert.deepStrictEqual(
+ process(`
+ // explains the button
+ <i18n.Translate>Confirm</i18n.Translate>`),
+ `#. screenid: 5
+#. explains the button
+#: test.tsx:6
+msgid "Confirm"
+msgstr ""`,
+ );
+});
+
+//
+// Duplicates.
+//
+
+test("should list every source reference of a repeated msgid", (t) => {
+ assert.deepStrictEqual(
+ processMany([
+ ["a.tsx", "t`Orders`;"],
+ ["b.tsx", "t`Orders`;"],
+ ]),
+ `#. screenid: 5
+#: a.tsx:4
+#: b.tsx:4
+msgid "Orders"
+msgstr ""`,
+ );
+});
+
+test("should list every source reference within one file", (t) => {
+ assert.deepStrictEqual(
+ process(`
+ <div>
+ <i18n.Translate>Orders</i18n.Translate>
+ <i18n.Translate>Orders</i18n.Translate>
+ </div>`),
+ `#. screenid: 5
+#: test.tsx:6
+#: test.tsx:7
+msgid "Orders"
+msgstr ""`,
+ );
+});
+
+test("the dedupe key must separate msgctxt from msgid", (t) => {
+ // msgctxt "ab" + msgid "c" must not collide with msgctxt "a" + msgid "bc".
+ assert.deepStrictEqual(
+ process(`i18n.context("ab")\`c\` + i18n.context("a")\`bc\`;`),
+ `#. screenid: 5
+#: test.tsx:4
+msgctxt "ab"
+msgid "c"
+msgstr ""
+
+#. screenid: 5
+#: test.tsx:4
+msgctxt "a"
+msgid "bc"
+msgstr ""`,
+ );
+});
+
+test("a repeated TranslateSwitch must not emit an orphan header", (t) => {
+ const out = process(`
+ <div>
+ <i18n.TranslateSwitch target={n}>
+ <i18n.TranslateSingular>one order</i18n.TranslateSingular>
+ <i18n.TranslatePlural>many orders</i18n.TranslatePlural>
+ </i18n.TranslateSwitch>
+ <i18n.TranslateSwitch target={n}>
+ <i18n.TranslateSingular>one order</i18n.TranslateSingular>
+ <i18n.TranslatePlural>many orders</i18n.TranslatePlural>
+ </i18n.TranslateSwitch>
+ </div>`);
+ assert.deepStrictEqual(
+ out,
+ `#. screenid: 5
+#: test.tsx:6
+#: test.tsx:10
+msgid "one order"
+msgid_plural "many orders"
+msgstr[0] ""
+msgstr[1] ""`,
+ );
+ // No "#:" block may be left without a msgid following it.
+ const lines = out.split("\n");
+ for (let i = 0; i < lines.length; i++) {
+ if (lines[i].startsWith("#: ")) {
+ assert.ok(
+ lines.slice(i).some((l) => l.startsWith("msgid ")),
+ "orphan source reference without a msgid",
+ );
+ }
+ }
+});
+
+//
+// i18n.plural, both the documented and the historic argument order.
+//
+
+test("should accept i18n.plural(count, singular, plural) as documented", (t) => {
+ assert.deepStrictEqual(
+ process(
+ "i18n.plural(n, i18n.lazy`I have ${n} apple`, i18n.lazy`I have ${n} apples`);",
+ ),
+ `#. screenid: 5
+#: test.tsx:4
+msgid "I have %1$s apple"
+msgid_plural "I have %1$s apples"
+msgstr[0] ""
+msgstr[1] ""`,
+ );
+});
+
+test("should accept i18n.plural(singular, plural)", (t) => {
+ assert.deepStrictEqual(
+ process(
+ "i18n.plural(i18n.lazy`I have ${n} apple`, i18n.lazy`I have ${n} apples`);",
+ ),
+ `#. screenid: 5
+#: test.tsx:4
+msgid "I have %1$s apple"
+msgid_plural "I have %1$s apples"
+msgstr[0] ""
+msgstr[1] ""`,
+ );
+});
+
+test("should not crash on an i18n.plural with too few arguments", (t) => {
+ assert.deepStrictEqual(process("i18n.plural();"), "");
+ assert.deepStrictEqual(process("i18n.plural(n);"), "");
+ // Degenerate, but must not throw: the lone tagged template is still picked
+ // up by the ordinary tagged-template rule.
+ assert.deepStrictEqual(
+ process("i18n.plural(i18n.lazy`one`);"),
+ `#. screenid: 5
+#: test.tsx:4
+msgid "one"
+msgstr ""`,
+ );
+});
+
+//
+// Error handling.
+//
+
+test("an empty template is a ParseError with a one-based line", (t) => {
+ assert.throws(
+ () => process("i18n.str``;"),
+ (e: unknown) => {
+ assert.ok(e instanceof ParseError, `not a ParseError: ${e}`);
+ assert.deepStrictEqual(e.line, 3);
+ assert.ok(
+ e.message.includes("test.tsx:4"),
+ `expected a one-based line in ${e.message}`,
+ );
+ return true;
+ },
+ );
+});
+
+test("class name is ParseError, not ParseErrror", (t) => {
+ assert.deepStrictEqual(ParseError.name, "ParseError");
+});
+
+test("should not crash on a shorthand boolean JSX attribute", (t) => {
+ assert.deepStrictEqual(
+ process(`<i18n.Translate disabled>something</i18n.Translate>`),
+ `#. screenid: 5
+#: test.tsx:4
+msgid "something"
+msgstr ""`,
+ );
+});
+
+test("should not crash on a spread JSX attribute", (t) => {
+ assert.deepStrictEqual(
+ process(`<i18n.Translate {...props}>something</i18n.Translate>`),
+ `#. screenid: 5
+#: test.tsx:4
+msgid "something"
+msgstr ""`,
+ );
+});
diff --git a/packages/pogen/src/potextract.ts b/packages/pogen/src/potextract.ts
@@ -54,7 +54,10 @@ function getTemplate(node: ts.Node): string {
let textFragments = [te.head.text];
for (let tsp of te.templateSpans) {
textFragments.push(`%${(textFragments.length - 1) / 2 + 1}$s`);
- textFragments.push(tsp.literal.text.replace(/%/g, "%%"));
+ // No escaping of "%" here: the runtime is not a printf formatter, it
+ // substitutes "%N$s" by literal string replacement over the *raw*
+ // template parts. An escaped msgid could never be looked up again.
+ textFragments.push(tsp.literal.text);
}
return textFragments.join("");
}
@@ -69,41 +72,65 @@ function getComment(
lastTokLine: number,
node: ts.Node,
): string {
- let lc = ts.getLineAndCharacterOfPosition(sourceFile, node.pos);
- let lastComments: ts.CommentRange[] | undefined = undefined;
+ let lc = ts.getLineAndCharacterOfPosition(
+ sourceFile,
+ node.getStart(sourceFile),
+ );
+ const found: ts.CommentRange[] = [];
for (let l = preLastTokLine; l < lastTokLine; l++) {
let pos = ts.getPositionOfLineAndCharacter(sourceFile, l, 0);
let comments = ts.getTrailingCommentRanges(sourceFile.text, pos);
- if (comments) {
- lastComments = comments;
+ if (!comments) {
+ continue;
+ }
+ for (const c of comments) {
+ if (found.length && found[found.length - 1].pos >= c.pos) {
+ continue;
+ }
+ found.push(c);
}
}
- if (!lastComments) {
+ if (!found.length) {
return "";
}
- let candidate = lastComments[lastComments.length - 1];
- let candidateEndLine = ts.getLineAndCharacterOfPosition(
- sourceFile,
- candidate.end,
- ).line;
- if (candidateEndLine != lc.line - 1) {
+ const startLineOf = (c: ts.CommentRange) =>
+ ts.getLineAndCharacterOfPosition(sourceFile, c.pos).line;
+ const endLineOf = (c: ts.CommentRange) =>
+ ts.getLineAndCharacterOfPosition(sourceFile, c.end).line;
+
+ let first = found.length - 1;
+ if (endLineOf(found[first]) != lc.line - 1) {
return "";
}
- let text = sourceFile.text.slice(candidate.pos, candidate.end);
- switch (candidate.kind) {
- case ts.SyntaxKind.SingleLineCommentTrivia:
- // Remove comment leader
- text = text.replace(/^[/][/]\s*/, "");
- break;
- case ts.SyntaxKind.MultiLineCommentTrivia:
- // Remove comment leader and trailer,
- // handling white space just like xgettext.
- text = text
- .replace(/^[/][*](\s*?\n|\s*)?/, "")
- .replace(/(\n[ \t]*?)?[*][/]$/, "");
- break;
+ // A run of consecutive "//" lines is one comment, just like in xgettext.
+ if (found[first].kind === ts.SyntaxKind.SingleLineCommentTrivia) {
+ while (
+ first > 0 &&
+ found[first - 1].kind === ts.SyntaxKind.SingleLineCommentTrivia &&
+ endLineOf(found[first - 1]) === startLineOf(found[first]) - 1
+ ) {
+ first--;
+ }
}
- return text;
+ const lines: string[] = [];
+ for (const candidate of found.slice(first)) {
+ let text = sourceFile.text.slice(candidate.pos, candidate.end);
+ switch (candidate.kind) {
+ case ts.SyntaxKind.SingleLineCommentTrivia:
+ // Remove comment leader
+ text = text.replace(/^[/][/]\s*/, "");
+ break;
+ case ts.SyntaxKind.MultiLineCommentTrivia:
+ // Remove comment leader and trailer,
+ // handling white space just like xgettext.
+ text = text
+ .replace(/^[/][*](\s*?\n|\s*)?/, "")
+ .replace(/(\n[ \t]*?)?[*][/]$/, "");
+ break;
+ }
+ lines.push(text);
+ }
+ return lines.join("\n");
}
function getPath(node: ts.Node): { path: string[]; ctx: string } {
@@ -189,55 +216,118 @@ function processTaggedTemplateExpression(
type ScreenInfo = {
fileMap: Record<string, Set<string>>;
- missing: Set<string>;
maxId: number;
};
const SCREEN_INFO: ScreenInfo = {
fileMap: {},
- missing: new Set(),
maxId: 0,
};
-function formatScreenId(
+function registerScreenId(
sourceFile: ts.SourceFile,
- outChunks: string[],
screenId: string | undefined,
-): void {
+): string[] {
if (!screenId) {
- return;
+ return [];
}
const screen = Number.parseInt(screenId, 10);
if (!screen || Number.isNaN(screen)) {
- SCREEN_INFO.missing.add(sourceFile.fileName);
- } else {
- if (!SCREEN_INFO.fileMap[screenId]) {
- SCREEN_INFO.fileMap[screenId] = new Set();
- }
- SCREEN_INFO.fileMap[screenId].add(sourceFile.fileName);
-
- if (SCREEN_INFO.maxId < screen) {
- SCREEN_INFO.maxId = screen;
- }
+ return [];
+ }
+ if (!SCREEN_INFO.fileMap[screenId]) {
+ SCREEN_INFO.fileMap[screenId] = new Set();
+ }
+ SCREEN_INFO.fileMap[screenId].add(sourceFile.fileName);
- outChunks.push(`#. screenid: ${screenId}\n`);
+ if (SCREEN_INFO.maxId < screen) {
+ SCREEN_INFO.maxId = screen;
}
+
+ return [screenId];
+}
+
+/**
+ * One message of the catalogue. Messages are buffered instead of being
+ * written out directly, so that every source reference of a msgid that occurs
+ * more than once can be listed together, the way xgettext does it.
+ */
+interface PoEntry {
+ screenIds: string[];
+ comments: string[];
+ refs: string[];
+ context: string;
+ msgid: string;
+ msgidPlural?: string;
}
-function formatMsgComment(
+/** Buffered messages, keyed by msgctxt+msgid, in first-seen order. */
+export type PoEntries = Map<string, PoEntry>;
+
+/**
+ * Dedupe key. The EOT separator is the gettext convention; without it
+ * msgctxt "ab"+msgid "c" would collide with msgctxt "a"+msgid "bc".
+ */
+function msgKey(context: string, msgid: string): string {
+ return `${context}${msgid}`;
+}
+
+function sourceRef(
projectPrefix: string,
sourceFile: ts.SourceFile,
- outChunks: string[],
line: number,
- comment?: string,
-) {
- if (comment) {
- for (let cl of comment.split("\n")) {
- outChunks.push(`#. ${cl}\n`);
+): string {
+ const fn = path.relative(projectPrefix, sourceFile.fileName);
+ return `${fn}:${line + 1}`;
+}
+
+function addEntry(entries: PoEntries, entry: PoEntry): void {
+ const key = msgKey(entry.context, entry.msgid);
+ const known = entries.get(key);
+ if (!known) {
+ entries.set(key, entry);
+ return;
+ }
+ const merge = (into: string[], from: string[]) => {
+ for (const x of from) {
+ if (!into.includes(x)) {
+ into.push(x);
+ }
}
+ };
+ merge(known.screenIds, entry.screenIds);
+ merge(known.comments, entry.comments);
+ merge(known.refs, entry.refs);
+ if (!known.msgidPlural && entry.msgidPlural) {
+ known.msgidPlural = entry.msgidPlural;
}
- const fn = path.relative(projectPrefix, sourceFile.fileName);
- outChunks.push(`#: ${fn}:${line + 1}\n`);
- outChunks.push(`#, c-format\n`);
+}
+
+function renderEntries(entries: PoEntries): string {
+ const outChunks: string[] = [];
+ for (const entry of entries.values()) {
+ for (const screenId of entry.screenIds) {
+ outChunks.push(`#. screenid: ${screenId}\n`);
+ }
+ for (const comment of entry.comments) {
+ for (let cl of comment.split("\n")) {
+ outChunks.push(`#. ${cl}\n`);
+ }
+ }
+ for (const ref of entry.refs) {
+ outChunks.push(`#: ${ref}\n`);
+ }
+ formatMsgLine(outChunks, "msgctxt", entry.context);
+ formatMsgLine(outChunks, "msgid", entry.msgid);
+ if (entry.msgidPlural !== undefined) {
+ formatMsgLine(outChunks, "msgid_plural", entry.msgidPlural);
+ outChunks.push(`msgstr[0] ""\n`);
+ outChunks.push(`msgstr[1] ""\n`);
+ } else {
+ outChunks.push(`msgstr ""\n`);
+ }
+ outChunks.push("\n");
+ }
+ return outChunks.join("");
}
function formatMsgLine(outChunks: string[], head: string, msg: string) {
@@ -299,12 +389,18 @@ function getJsxAttribute(sour: ts.SourceFile, node: ts.Node) {
case ts.SyntaxKind.JsxOpeningElement: {
let e = childNode as ts.JsxOpeningElement;
- e.attributes.properties.map((p) => {
- const id = p.getChildAt(0, sour).getText(sour);
- const v = p.getChildAt(2, sour);
- if (v.kind !== ts.SyntaxKind.StringLiteral) {
- return undefined;
+ e.attributes.properties.forEach((p) => {
+ // Skip spreads ({...props}) and shorthand boolean attributes
+ // (<i18n.Translate disabled>), which have no initializer at all.
+ if (p.kind !== ts.SyntaxKind.JsxAttribute) {
+ return;
}
+ const attr = p as ts.JsxAttribute;
+ const v = attr.initializer;
+ if (!v || v.kind !== ts.SyntaxKind.StringLiteral) {
+ return;
+ }
+ const id = attr.name.getText(sour);
result[id] = JSON.parse(v.getText(sour));
});
return;
@@ -348,7 +444,6 @@ function getJsxContent(node: ts.Node) {
case ts.SyntaxKind.JsxClosingElement:
break;
default: {
- console.log("unhandled node type: ", childNode.kind);
let lc = ts.getLineAndCharacterOfPosition(
childNode.getSourceFile(),
childNode.getStart(),
@@ -444,27 +539,104 @@ function searchScreenId(parents: ts.Node[], sourceFile: ts.SourceFile) {
return result;
}
+/**
+ * Report a call site that looks like it wants to be translated but can never
+ * be extracted. Non-fatal on purpose: the point is that these stop being
+ * invisible.
+ */
+function warnUnextractableCall(
+ sourceFile: ts.SourceFile,
+ line: number,
+ form: string,
+ detail: string,
+) {
+ console.error(
+ `pogen: warning: ${sourceFile.fileName}:${line + 1}: ${form} is not extractable, ${detail}`,
+ );
+}
+
+function checkUnextractableCall(
+ parents: ts.Node[],
+ ce: ts.CallExpression,
+ callee: string[],
+ sourceFile: ts.SourceFile,
+ line: number,
+) {
+ if (callee[0] !== "i18n" && callee[0] !== "t") {
+ return;
+ }
+ // i18n.context("x")`y` is a context specifier for the tagged template that
+ // follows, not a message of its own.
+ const parent = parents[0];
+ if (
+ parent !== undefined &&
+ parent.kind === ts.SyntaxKind.TaggedTemplateExpression &&
+ (parent as ts.TaggedTemplateExpression).tag === ce
+ ) {
+ return;
+ }
+ if (ce.arguments.length !== 1) {
+ return;
+ }
+ const arg = ce.arguments[0];
+ const name = callee.join(".");
+ switch (arg.kind) {
+ case ts.SyntaxKind.StringLiteral:
+ case ts.SyntaxKind.NoSubstitutionTemplateLiteral:
+ warnUnextractableCall(
+ sourceFile,
+ line,
+ `${name}(<string literal>)`,
+ `use a tagged template ${name}\`...\` instead`,
+ );
+ break;
+ case ts.SyntaxKind.TemplateExpression:
+ warnUnextractableCall(
+ sourceFile,
+ line,
+ `${name}(\`...\${...}...\`)`,
+ `it mints a new msgid for every runtime value; use a tagged template ${name}\`...\` instead`,
+ );
+ break;
+ }
+}
+
function processNode(
parents: ts.Node[],
node: ts.Node,
preLastTokLine: number,
lastTokLine: number,
sourceFile: ts.SourceFile,
- outChunks: string[],
- knownMessageIds: Set<string>,
+ entries: PoEntries,
projectPrefix: string,
) {
- const { line } = ts.getLineAndCharacterOfPosition(sourceFile, node.pos);
+ const { line } = ts.getLineAndCharacterOfPosition(
+ sourceFile,
+ node.getStart(sourceFile),
+ );
try {
switch (node.kind) {
case ts.SyntaxKind.JsxElement: {
let path = getJsxElementPath(node);
+ if (
+ arrayEq(path, ["i18n", "Translate"]) ||
+ arrayEq(path, ["i18n", "TranslateSwitch"])
+ ) {
+ // Track the line of the element itself, so that getComment has a
+ // range to scan. Without this a JSX message never sees a comment.
+ if (line != lastTokLine) {
+ preLastTokLine = lastTokLine;
+ lastTokLine = line;
+ }
+ }
// <i18n.Translate>text</i18n.Translate>
if (arrayEq(path, ["i18n", "Translate"])) {
const content = getJsxContent(node);
if (!content) {
- throw Error(
- `string to be translated can't be empty: ${sourceFile.fileName}:${line}`,
+ throw new ParseError(
+ `string to be translated can't be empty`,
+ sourceFile.fileName,
+ line,
);
}
const comment = getComment(
@@ -474,78 +646,85 @@ function processNode(
node,
);
const context = getJsxAttribute(sourceFile, node)["context"] ?? "";
- const msgid = context + content;
- if (!knownMessageIds.has(msgid)) {
- knownMessageIds.add(msgid);
- const screenId = searchScreenId(parents, sourceFile);
- formatScreenId(sourceFile, outChunks, screenId);
- formatMsgComment(
- projectPrefix,
+ addEntry(entries, {
+ screenIds: registerScreenId(
sourceFile,
- outChunks,
- line,
- comment,
- );
- formatMsgLine(outChunks, "msgctxt", context);
- formatMsgLine(outChunks, "msgid", content);
- outChunks.push(`msgstr ""\n`);
- outChunks.push("\n");
- }
+ searchScreenId(parents, sourceFile),
+ ),
+ comments: comment ? [comment] : [],
+ refs: [sourceRef(projectPrefix, sourceFile, line)],
+ context,
+ msgid: content,
+ });
} else if (arrayEq(path, ["i18n", "TranslateSwitch"])) {
- const { line } = ts.getLineAndCharacterOfPosition(
- sourceFile,
- node.pos,
- );
const comment = getComment(
sourceFile,
preLastTokLine,
lastTokLine,
node,
);
- formatMsgComment(projectPrefix, sourceFile, outChunks, line, comment);
const content = getJsxSingular(node);
if (!content) {
- throw Error(
- `string to be translated can't be empty, singular is missing: ${sourceFile.fileName}:${line}`,
+ throw new ParseError(
+ `string to be translated can't be empty, singular is missing`,
+ sourceFile.fileName,
+ line,
);
}
const pluralForm = getJsxPlural(node);
if (!pluralForm) {
- throw Error(
- `string to be translated can't be empty, plural is missing: ${sourceFile.fileName}:${line}`,
+ throw new ParseError(
+ `string to be translated can't be empty, plural is missing`,
+ sourceFile.fileName,
+ line,
);
}
const context = getJsxAttribute(sourceFile, node)["context"] ?? "";
- const msgid = context + content;
- if (!knownMessageIds.has(msgid)) {
- knownMessageIds.add(msgid);
- const screenId = searchScreenId(parents, sourceFile);
- formatScreenId(sourceFile, outChunks, screenId);
- formatMsgLine(outChunks, "msgctxt", context);
- formatMsgLine(outChunks, "msgid", content);
- formatMsgLine(outChunks, "msgid_plural", pluralForm);
- outChunks.push(`msgstr[0] ""\n`);
- outChunks.push(`msgstr[1] ""\n`);
- outChunks.push(`\n`);
- }
+ addEntry(entries, {
+ screenIds: registerScreenId(
+ sourceFile,
+ searchScreenId(parents, sourceFile),
+ ),
+ comments: comment ? [comment] : [],
+ refs: [sourceRef(projectPrefix, sourceFile, line)],
+ context,
+ msgid: content,
+ msgidPlural: pluralForm,
+ });
}
break;
}
case ts.SyntaxKind.CallExpression: {
- // might be i18n.plural(i18n[.X]`...`, i18n[.X]`...`)
+ // might be i18n.plural(n?, i18n[.X]`...`, i18n[.X]`...`)
let ce = <ts.CallExpression>node;
let path = getPath(ce.expression);
if (!arrayEq(path.path, ["i18n", "plural"])) {
+ checkUnextractableCall(parents, ce, path.path, sourceFile, line);
+ break;
+ }
+ // The README documents i18n.plural(n, i18n.lazy`..`, i18n.lazy`..`);
+ // older code passes just the two forms. Accept both.
+ let first = 0;
+ if (
+ ce.arguments.length > 0 &&
+ ce.arguments[0].kind != ts.SyntaxKind.TaggedTemplateExpression
+ ) {
+ first = 1;
+ }
+ if (ce.arguments.length < first + 2) {
break;
}
- if (ce.arguments[0].kind != ts.SyntaxKind.TaggedTemplateExpression) {
+ if (
+ ce.arguments[first].kind != ts.SyntaxKind.TaggedTemplateExpression
+ ) {
break;
}
- if (ce.arguments[1].kind != ts.SyntaxKind.TaggedTemplateExpression) {
+ if (
+ ce.arguments[first + 1].kind != ts.SyntaxKind.TaggedTemplateExpression
+ ) {
break;
}
- let { line } = ts.getLineAndCharacterOfPosition(sourceFile, ce.pos);
- const tte1 = <ts.TaggedTemplateExpression>ce.arguments[0];
+ const tte1 = <ts.TaggedTemplateExpression>ce.arguments[first];
let lc1 = ts.getLineAndCharacterOfPosition(sourceFile, tte1.pos);
if (lc1.line != lastTokLine) {
preLastTokLine = lastTokLine; // HERE
@@ -559,14 +738,14 @@ function processNode(
);
const content = t1.template;
if (!content) {
- throw new ParseErrror(
+ throw new ParseError(
`string to be translated can't be empty`,
sourceFile.fileName,
line,
);
}
- const tte2 = <ts.TaggedTemplateExpression>ce.arguments[1];
+ const tte2 = <ts.TaggedTemplateExpression>ce.arguments[first + 1];
let lc2 = ts.getLineAndCharacterOfPosition(sourceFile, tte2.pos);
if (lc2.line != lastTokLine) {
preLastTokLine = lastTokLine; // HERE
@@ -579,19 +758,17 @@ function processNode(
tte2,
);
let comment = getComment(sourceFile, preLastTokLine, lastTokLine, ce);
- const msgid = path.ctx + content;
- if (!knownMessageIds.has(msgid)) {
- knownMessageIds.add(msgid);
- const screenId = searchScreenId(parents, sourceFile);
- formatScreenId(sourceFile, outChunks, screenId);
- formatMsgComment(projectPrefix, sourceFile, outChunks, line, comment);
- formatMsgLine(outChunks, "msgctxt", path.ctx);
- formatMsgLine(outChunks, "msgid", content);
- formatMsgLine(outChunks, "msgid_plural", t2.template);
- outChunks.push(`msgstr[0] ""\n`);
- outChunks.push(`msgstr[1] ""\n`);
- outChunks.push("\n");
- }
+ addEntry(entries, {
+ screenIds: registerScreenId(
+ sourceFile,
+ searchScreenId(parents, sourceFile),
+ ),
+ comments: comment ? [comment] : [],
+ refs: [sourceRef(projectPrefix, sourceFile, line)],
+ context: path.ctx,
+ msgid: content,
+ msgidPlural: t2.template,
+ });
// Important: no processing for child i18n expressions here
return;
@@ -603,34 +780,38 @@ function processNode(
preLastTokLine = lastTokLine;
lastTokLine = lc2.line;
}
- const { comment, template, line, path, context } =
- processTaggedTemplateExpression(
- sourceFile,
- preLastTokLine,
- lastTokLine,
- tte,
- );
- if (path[0] != "i18n") {
+ const {
+ comment,
+ template,
+ line: tteLine,
+ path,
+ context,
+ } = processTaggedTemplateExpression(
+ sourceFile,
+ preLastTokLine,
+ lastTokLine,
+ tte,
+ );
+ if (path[0] != "i18n" && path[0] != "t") {
break;
}
if (!template) {
- throw new ParseErrror(
+ throw new ParseError(
`string to be translated can't be empty`,
sourceFile.fileName,
- line,
+ tteLine,
);
}
- const msgid = context + template;
- if (!knownMessageIds.has(msgid)) {
- knownMessageIds.add(msgid);
- const screenId = searchScreenId(parents, sourceFile);
- formatScreenId(sourceFile, outChunks, screenId);
- formatMsgComment(projectPrefix, sourceFile, outChunks, line, comment);
- formatMsgLine(outChunks, "msgctxt", context);
- formatMsgLine(outChunks, "msgid", template);
- outChunks.push(`msgstr ""\n`);
- outChunks.push("\n");
- }
+ addEntry(entries, {
+ screenIds: registerScreenId(
+ sourceFile,
+ searchScreenId(parents, sourceFile),
+ ),
+ comments: comment ? [comment] : [],
+ refs: [sourceRef(projectPrefix, sourceFile, tteLine)],
+ context,
+ msgid: template,
+ });
break;
}
}
@@ -639,59 +820,53 @@ function processNode(
processNode(
[node, ...parents],
child,
- lastTokLine,
preLastTokLine,
+ lastTokLine,
sourceFile,
- outChunks,
- knownMessageIds,
+ entries,
projectPrefix,
);
});
} catch (error) {
- if (error instanceof ParseErrror) {
+ if (error instanceof ParseError) {
throw error;
} else {
- throw new ParseErrror(String(error), sourceFile.fileName, line);
+ throw new ParseError(String(error), sourceFile.fileName, line);
}
}
}
-class ParseErrror extends Error {
+export class ParseError extends Error {
+ /** Zero-based, like everything else the TypeScript API hands out. */
line: number;
filename: string;
reason: string;
constructor(reason: string, filename: string, line: number) {
- super(`failed by "${reason}" at ${filename}:${line}`);
+ // Reported one-based, so that it agrees with the "#:" lines and with
+ // what an editor shows.
+ super(`failed by "${reason}" at ${filename}:${line + 1}`);
this.reason = reason;
this.filename = filename;
this.line = line;
}
}
-export function processFileForTesting(sourceFile: ts.SourceFile): string {
- const result: string[] = new Array<string>();
- processNode([], sourceFile, 0, 0, sourceFile, result, new Set<string>(), "");
- return result.join("");
+export function processFileForTesting(
+ ...sourceFiles: ts.SourceFile[]
+): string {
+ const entries: PoEntries = new Map();
+ for (const sourceFile of sourceFiles) {
+ processFile(sourceFile, entries, "");
+ }
+ return renderEntries(entries);
}
export function processFile(
sourceFile: ts.SourceFile,
- outChunks: string[],
- knownMessageIds: Set<string>,
+ entries: PoEntries,
projectPrefix: string,
) {
- // let lastTokLine = 0;
- // let preLastTokLine = 0;
- processNode(
- [],
- sourceFile,
- 0,
- 0,
- sourceFile,
- outChunks,
- knownMessageIds,
- projectPrefix,
- );
+ processNode([], sourceFile, 0, 0, sourceFile, entries, projectPrefix);
}
function searchIntoParents(directory: string, fileFlag: string) {
@@ -758,21 +933,27 @@ export function potextract(searchPath: string = "./") {
const gitRoot = searchIntoParents(process.cwd(), ".git");
- const chunks = [header];
- const knownMessageIds = new Set<string>();
+ const entries: PoEntries = new Map();
+ const failed: string[] = [];
for (const f of ownFiles) {
- processFile(f, chunks, knownMessageIds, gitRoot);
+ // One unparseable file must not cost us the whole catalogue.
+ try {
+ processFile(f, entries, gitRoot);
+ } catch (e) {
+ failed.push(f.fileName);
+ if (e instanceof ParseError) {
+ console.error(
+ `pogen: error: ${e.filename}:${e.line + 1}: ${e.reason} (file skipped)`,
+ );
+ } else {
+ console.error(
+ `pogen: error: ${f.fileName}: ${String(e)} (file skipped)`,
+ );
+ }
+ }
}
- if (SCREEN_INFO.missing.size) {
- console.error(
- `There are some files with translation strings that do not have a constant TALER_SCREEN_ID on the root. This constant should be a unique integer number to facilitate the identification from other locations. Add a "const TALER_SCREEN_ID = <number>;" after the last import. It does not need to be exported and it can be removed from runtime. Files:`,
- );
- SCREEN_INFO.missing.forEach((fileName) => {
- console.error(` * ${fileName}`);
- });
- }
const haveRepetition =
Object.values(SCREEN_INFO.fileMap).find((files) => files.size > 1) !==
undefined;
@@ -791,9 +972,7 @@ export function potextract(searchPath: string = "./") {
});
}
- const pot = chunks.join("");
-
- //console.log(pot);
+ const pot = header + renderEntries(entries);
const packageJson = JSON.parse(
fs.readFileSync("./package.json", { encoding: "utf-8" }),
@@ -805,4 +984,11 @@ export function potextract(searchPath: string = "./") {
process.exit(1);
}
fs.writeFileSync(`./src/i18n/${poDomain}.pot`, pot);
+
+ if (failed.length) {
+ console.error(
+ `pogen: ${failed.length} file(s) could not be processed, the catalogue is incomplete`,
+ );
+ process.exitCode = 1;
+ }
}
