add translation completeness from pogen to the UI

9 files changed, 231 insertions, 75 deletions

diff --git a/packages/demobank-ui/src/components/app.tsx b/packages/demobank-ui/src/components/app.tsx
index 3d1a43803..c3e579810 100644
--- a/packages/demobank-ui/src/components/app.tsx
+++ b/packages/demobank-ui/src/components/app.tsx
@@ -24,7 +24,7 @@ import { Fragment, FunctionalComponent, h } from "preact";
 import { SWRConfig } from "swr";
 import { BackendStateProvider } from "../context/backend.js";
 import { BankCoreApiProvider } from "../context/config.js";
-import { strings } from "../i18n/strings.js";
+import { strings, StringsType } from "../i18n/strings.js";
 import { BankUiSettings, fetchSettings } from "../settings.js";
 import { Routing } from "../Routing.js";
 import { BankFrame } from "../pages/BankFrame.js";
@@ -42,7 +42,7 @@ const App: FunctionalComponent = () => {
   const baseUrl = getInitialBackendBaseURL(settings.backendBaseURL);
   return (
     <SettingsProvider value={settings}>
-      <TranslationProvider source={strings}>
+      <TranslationProvider source={strings} completness={{ "es": strings["es"].completeness, "de": strings["de"].completeness }}>
         <BackendStateProvider>
           <BankCoreApiProvider baseUrl={baseUrl} frameOnError={BankFrame}>
             <SWRConfig
diff --git a/packages/demobank-ui/src/i18n/strings-prelude b/packages/demobank-ui/src/i18n/strings-prelude
deleted file mode 100644
index a0aeb8268..000000000
--- a/packages/demobank-ui/src/i18n/strings-prelude
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- This file is part of GNU Taler
- (C) 2022 Taler Systems S.A.
-
- GNU Taler is free software; you can redistribute it and/or modify it under the
- terms of the GNU General Public License as published by the Free Software
- Foundation; either version 3, or (at your option) any later version.
-
- GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY
- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along with
- GNU Taler; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
- */
-
-/*eslint quote-props: ["error", "consistent"]*/
-export const strings: {[s: string]: any} = {};
-
diff --git a/packages/demobank-ui/src/i18n/strings.ts b/packages/demobank-ui/src/i18n/strings.ts
index fada43b38..ddff053eb 100644
--- a/packages/demobank-ui/src/i18n/strings.ts
+++ b/packages/demobank-ui/src/i18n/strings.ts
@@ -1,24 +1,17 @@
-/*
- This file is part of GNU Taler
- (C) 2022 Taler Systems S.A.
 
- GNU Taler is free software; you can redistribute it and/or modify it under the
- terms of the GNU General Public License as published by the Free Software
- Foundation; either version 3, or (at your option) any later version.
-
- GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY
- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along with
- GNU Taler; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
- */
-
-/*eslint quote-props: ["error", "consistent"]*/
-export const strings: {[s: string]: any} = {};
+export interface StringsType {
+  // X-Domain or 'messages'
+  domain: string;
+  lang: string;
+  completeness: number,
+  'plural_forms': string;
+  locale_data: {
+    messages: Record<string, any>
+  }
+}
+export const strings: Record<string,StringsType> = {};
 
 strings['it'] = {
-  "domain": "messages",
   "locale_data": {
     "messages": {
       "": {
@@ -1062,11 +1055,14 @@ strings['it'] = {
         ""
       ]
     }
-  }
+  },
+  "domain": "messages",
+  "plural_forms": "nplurals=2; plural=n != 1;",
+  "lang": "it",
+  "completeness": 14
 };
 
 strings['fr'] = {
-  "domain": "messages",
   "locale_data": {
     "messages": {
       "": {
@@ -2110,11 +2106,14 @@ strings['fr'] = {
         ""
       ]
     }
-  }
+  },
+  "domain": "messages",
+  "plural_forms": "nplurals=2; plural=n > 1;",
+  "lang": "fr",
+  "completeness": 0
 };
 
 strings['es'] = {
-  "domain": "messages",
   "locale_data": {
     "messages": {
       "": {
@@ -3158,11 +3157,14 @@ strings['es'] = {
         "Bienvenido a %1$s!"
       ]
     }
-  }
+  },
+  "domain": "messages",
+  "plural_forms": "nplurals=2; plural=n != 1;",
+  "lang": "es",
+  "completeness": 100
 };
 
 strings['en'] = {
-  "domain": "messages",
   "locale_data": {
     "messages": {
       "": {
@@ -4206,11 +4208,14 @@ strings['en'] = {
         ""
       ]
     }
-  }
+  },
+  "domain": "messages",
+  "plural_forms": "nplurals=2; plural=(n != 1);",
+  "lang": "en",
+  "completeness": 100
 };
 
 strings['de'] = {
-  "domain": "messages",
   "locale_data": {
     "messages": {
       "": {
@@ -5254,6 +5259,10 @@ strings['de'] = {
         ""
       ]
     }
-  }
+  },
+  "domain": "messages",
+  "plural_forms": "nplurals=2; plural=n != 1;",
+  "lang": "de",
+  "completeness": 4
 };
 
diff --git a/packages/pogen/src/po2ts.ts b/packages/pogen/src/po2ts.ts
index d37bdb902..0e2a0d6ea 100644
--- a/packages/pogen/src/po2ts.ts
+++ b/packages/pogen/src/po2ts.ts
@@ -19,12 +19,53 @@
  */
 
 // @ts-ignore
-import * as po2json from "po2json";
+import * as po2jsonLib from "po2json";
 import * as fs from "fs";
-import * as path from "path";
 import glob = require("glob");
 
-const DEFAULT_STRING_PRELUDE = "export const strings: any = {};\n\n"
+//types defined by the po2json library
+type Header = {
+  domain: string;
+  lang: string;
+  'plural_forms': string;
+};
+
+type MessagesType = Record<string, undefined | Array<string>> & { "": Header }
+interface pojsonType {
+  // X-Domain or 'messages'
+  domain: string;
+  locale_data: {
+    messages: MessagesType
+  }
+}
+// ----------- end pf po2json
+
+interface StringsType {
+  // X-Domain or 'messages'
+  domain: string;
+  lang: string;
+  completeness: number,
+  'plural_forms': string;
+  locale_data: {
+    messages: Record<string, undefined | Array<string>>
+  }
+}
+
+// This prelude match the types above
+const TYPES_FOR_STRING_PRELUDE = `
+export interface StringsType {
+  domain: string;
+  lang: string;
+  completeness: number;
+  'plural_forms': string;
+  locale_data: {
+    messages: Record<string, any>;
+  };
+};
+`;
+
+const DEFAULT_STRING_PRELUDE = `${TYPES_FOR_STRING_PRELUDE}export const strings: Record<string,StringsType> = {};\n\n`
+
 
 export function po2ts(): void {
   const files = glob.sync("src/i18n/*.po");
@@ -54,16 +95,26 @@ export function po2ts(): void {
     }
 
     const lang = m[1];
-    const pojson = po2json.parseFileSync(filename, {
+    const poAsJson: pojsonType = po2jsonLib.parseFileSync(filename, {
       format: "jed1.x",
       fuzzy: true,
     });
-    const s =
-      "strings['" +
-      lang +
-      "'] = " +
-      JSON.stringify(pojson, null, "  ") +
-      ";\n\n";
+    const header = poAsJson.locale_data.messages[""]
+    const total = calculateTotalTranslations(poAsJson.locale_data.messages)
+    const completeness =
+      header.lang === "en"
+        ? 100 // 'en' is always complete
+        : Math.floor(total.translations * 100 / total.keys);
+
+    const strings: StringsType = {
+      locale_data: poAsJson.locale_data,
+      domain: poAsJson.domain,
+      plural_forms: header.plural_forms,
+      lang: header.lang,
+      completeness,
+    }
+    const value = JSON.stringify(strings, undefined, 2)
+    const s = `strings['${lang}'] = ${value};\n\n`
     chunks.push(s);
   }
 
@@ -71,3 +122,25 @@ export function po2ts(): void {
 
   fs.writeFileSync("src/i18n/strings.ts", tsContents);
 }
+
+function calculateTotalTranslations(msgs: MessagesType): { keys: number, translations: number } {
+  const kv = Object.entries(msgs)
+  const [keys, translations] = kv.reduce(([total, withTranslation], translation) => {
+    if (!translation || translation.length !== 2 || !translation[1]) {
+      //curent key is empty
+      return [total, withTranslation]
+    }
+    const v = translation[1]
+    if (!Array.isArray(v)) {
+      // this is not a translation
+      return [total, withTranslation]
+    }
+    if (!v.length || !v[0].length) {
+      //translation is missing
+      return [total + 1, withTranslation]
+    }
+    //current key has a translation
+    return [total + 1, withTranslation + 1]
+  }, [0, 0])
+  return { keys, translations }
+}
\ No newline at end of file
diff --git a/packages/web-util/src/components/Header.tsx b/packages/web-util/src/components/Header.tsx
index a0587b2ae..e5662fc70 100644
--- a/packages/web-util/src/components/Header.tsx
+++ b/packages/web-util/src/components/Header.tsx
@@ -3,7 +3,7 @@ import { LangSelector, useTranslationContext } from "../index.browser.js";
 import { ComponentChildren, Fragment, VNode, h } from "preact";
 import logo from "../assets/logo-2021.svg";
 
-export function Header({ title, iconLinkURL, sites, supportedLangs, onLogout, children }:
+export function Header({ title, iconLinkURL, sites, onLogout, children }:
   { title: string, iconLinkURL: string, children?: ComponentChildren, onLogout: (() => void) | undefined, sites: Array<Array<string>>, supportedLangs: string[] }): VNode {
   const { i18n } = useTranslationContext();
   const [open, setOpen] = useState(false)
@@ -107,7 +107,7 @@ export function Header({ title, iconLinkURL, sites, supportedLangs, onLogout, ch
                           </li>
                           : undefined}
                         <li>
-                          <LangSelector supportedLangs={supportedLangs} />
+                          <LangSelector />
                         </li>
                         {/* CHILDREN */}
                         {children}
diff --git a/packages/web-util/src/components/LangSelector.tsx b/packages/web-util/src/components/LangSelector.tsx
index a8d910129..7deaa0cf4 100644
--- a/packages/web-util/src/components/LangSelector.tsx
+++ b/packages/web-util/src/components/LangSelector.tsx
@@ -43,9 +43,9 @@ function getLangName(s: keyof LangsNames | string): string {
   return String(s);
 }
 
-export function LangSelector({ supportedLangs }: { supportedLangs: string[] }): VNode {
+export function LangSelector({ }: {}): VNode {
   const [updatingLang, setUpdatingLang] = useState(false);
-  const { lang, changeLanguage } = useTranslationContext();
+  const { lang, changeLanguage, completness, supportedLang } = useTranslationContext();
   const [hidden, setHidden] = useState(true);
 
   useEffect(() => {
@@ -66,8 +66,9 @@ export function LangSelector({ supportedLangs }: { supportedLangs: string[] }):
     <div>
       <div class="relative mt-2">
         <button type="button" class="relative w-full cursor-default rounded-md bg-white py-1.5 pl-3 pr-10 text-left text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 focus:outline-none focus:ring-2 focus:ring-indigo-600 sm:text-sm sm:leading-6" aria-haspopup="listbox" aria-expanded="true" aria-labelledby="listbox-label"
-          onClick={() => {
-            setHidden((h) => !h);
+          onClick={(e) => {
+            setHidden(!hidden);
+            e.stopPropagation()
           }}>
           <span class="flex items-center">
             <img alt="language" class="h-5 w-5 flex-shrink-0 rounded-full" src={langIcon} />
@@ -82,7 +83,7 @@ export function LangSelector({ supportedLangs }: { supportedLangs: string[] }):
 
         {!hidden &&
           <ul class="absolute z-10 mt-1 max-h-60 w-full overflow-auto rounded-md bg-white py-1 text-base shadow-lg ring-1 ring-black ring-opacity-5 focus:outline-none sm:text-sm" tabIndex={-1} role="listbox" aria-labelledby="listbox-label" aria-activedescendant="listbox-option-3">
-            {supportedLangs
+            {Object.keys(supportedLang)
               .filter((l) => l !== lang)
               .map((lang) => (
                 <li class="text-gray-900 hover:bg-indigo-600 hover:text-white cursor-pointer relative select-none py-2 pl-3 pr-9" role="option"
@@ -92,7 +93,10 @@ export function LangSelector({ supportedLangs }: { supportedLangs: string[] }):
                     setHidden(true)
                   }}
                 >
-                  <span class="font-normal block truncate">{getLangName(lang)}</span>
+                  <span class="font-normal truncate flex justify-between ">
+                    <span>{getLangName(lang)}</span>
+                    <span>{(completness as any)[lang]}%</span>
+                  </span>
 
                   <span class="text-indigo-600 absolute inset-y-0 right-0 flex items-center pr-4">
                     {/* <svg class="h-5 w-5" viewBox="0 0 20 20" fill="currentColor" aria-hidden="true">
diff --git a/packages/web-util/src/context/translation.ts b/packages/web-util/src/context/translation.ts
index fb6efc40a..2b9704939 100644
--- a/packages/web-util/src/context/translation.ts
+++ b/packages/web-util/src/context/translation.ts
@@ -34,6 +34,7 @@ interface Type {
   changeLanguage: (l: string) => void;
   i18n: InternationalizationAPI;
   dateLocale: Locale,
+  completness: { [id in keyof typeof supportedLang]: number }
 }
 
 const supportedLang = {
@@ -43,7 +44,6 @@ const supportedLang = {
   de: "Deutsch [de]",
   sv: "Svenska [sv]",
   it: "Italiane [it]",
-  navigator: "Defined by navigator",
 };
 
 const initial: Type = {
@@ -53,7 +53,15 @@ const initial: Type = {
     // do not change anything
   },
   i18n,
-  dateLocale: enLocale
+  dateLocale: enLocale,
+  completness: {
+    de: 0,
+    en: 0,
+    es: 0,
+    fr: 0,
+    it: 0,
+    sv: 0,
+  }
 };
 const Context = createContext<Type>(initial);
 
@@ -62,6 +70,7 @@ interface Props {
   children: ComponentChildren;
   forceLang?: string;
   source: Record<string, any>;
+  completness?: Record<string, number>;
 }
 
 // Outmost UI wrapper.
@@ -70,8 +79,17 @@ export const TranslationProvider = ({
   children,
   forceLang,
   source,
+  completness: completnessProp
 }: Props): VNode => {
-  const { value: lang, update: changeLanguage } = useLang(initial);
+  const completness = {
+    de: !completnessProp || !completnessProp["de"] ? 0 : completnessProp["de"],
+    en: !completnessProp || !completnessProp["en"] ? 0 : completnessProp["en"],
+    es: !completnessProp || !completnessProp["es"] ? 0 : completnessProp["es"],
+    fr: !completnessProp || !completnessProp["fr"] ? 0 : completnessProp["fr"],
+    it: !completnessProp || !completnessProp["it"] ? 0 : completnessProp["it"],
+    sv: !completnessProp || !completnessProp["sv"] ? 0 : completnessProp["sv"],
+  }
+  const { value: lang, update: changeLanguage } = useLang(initial, completness);
 
   useEffect(() => {
     if (forceLang) {
@@ -93,7 +111,7 @@ export const TranslationProvider = ({
         enLocale;
 
   return h(Context.Provider, {
-    value: { lang, changeLanguage, supportedLang, i18n, dateLocale },
+    value: { lang, changeLanguage, supportedLang, i18n, dateLocale, completness },
     children,
   });
 };
diff --git a/packages/web-util/src/hooks/useLang.ts b/packages/web-util/src/hooks/useLang.ts
index 448cd8aba..e4e512388 100644
--- a/packages/web-util/src/hooks/useLang.ts
+++ b/packages/web-util/src/hooks/useLang.ts
@@ -20,16 +20,42 @@ import {
   useLocalStorage,
 } from "./useLocalStorage.js";
 
-function getBrowserLang(): string | undefined {
+const MIN_LANG_COVERAGE_THRESHOLD = 90;
+/**
+ * choose the best from the browser config based on the completeness
+ * on the translation
+ */
+function getBrowserLang(completness: Record<string, number>): string | undefined {
   if (typeof window === "undefined") return undefined;
-  if (window.navigator.languages) return window.navigator.languages[0];
-  if (window.navigator.language) return window.navigator.language;
+
+  if (window.navigator.language) {
+    if (completness[window.navigator.language] >= MIN_LANG_COVERAGE_THRESHOLD) {
+      return window.navigator.language
+    }
+  }
+  if (window.navigator.languages) {
+    const match = Object.entries(completness).filter(([code, value]) => {
+      if (value < MIN_LANG_COVERAGE_THRESHOLD) return false; //do not consider langs below 90%
+      return window.navigator.languages.findIndex(l => l.startsWith(code)) !== -1
+    }).map(([code, value]) => ({ code, value }))
+
+    if (match.length > 0) {
+      let max = match[0]
+      match.forEach(v => {
+        if (v.value > max.value) {
+          max = v
+        }
+      })
+      return max.code
+    }
+  };
+
   return undefined;
 }
 
 const langPreferenceKey = buildStorageKey("lang-preference");
 
-export function useLang(initial?: string): Required<StorageState> {
-  const defaultValue = (getBrowserLang() || initial || "en").substring(0, 2);
+export function useLang(initial: string | undefined, completness: Record<string, number>): Required<StorageState> {
+  const defaultValue = (getBrowserLang(completness) || initial || "en").substring(0, 2);
   return useLocalStorage(langPreferenceKey, defaultValue);
 }
diff --git a/packages/web-util/src/index.build.ts b/packages/web-util/src/index.build.ts
index e2851dc3a..4a52d1177 100644
--- a/packages/web-util/src/index.build.ts
+++ b/packages/web-util/src/index.build.ts
@@ -121,6 +121,51 @@ const sassPlugin: esbuild.Plugin = {
   },
 };
 
+
+/**
+ * Problem: 
+ *   No loader is configured for ".node" files: ../../node_modules/.pnpm/fsevents@2.3.3/node_modules/fsevents/fsevents.node
+ * 
+ * Reference:
+ *   https://github.com/evanw/esbuild/issues/1051#issuecomment-806325487
+ */
+const nativeNodeModulesPlugin: esbuild.Plugin = {
+  name: 'native-node-modules',
+  setup(build) {
+    // If a ".node" file is imported within a module in the "file" namespace, resolve 
+    // it to an absolute path and put it into the "node-file" virtual namespace.
+    build.onResolve({ filter: /\.node$/, namespace: 'file' }, args => ({
+      path: require.resolve(args.path, { paths: [args.resolveDir] }),
+      namespace: 'node-file',
+    }))
+
+    // Files in the "node-file" virtual namespace call "require()" on the
+    // path from esbuild of the ".node" file in the output directory.
+    build.onLoad({ filter: /.*/, namespace: 'node-file' }, args => ({
+      contents: `
+        import path from ${JSON.stringify(args.path)}
+        try { module.exports = require(path) }
+        catch {}
+      `,
+    }))
+
+    // If a ".node" file is imported within a module in the "node-file" namespace, put
+    // it in the "file" namespace where esbuild's default loading behavior will handle
+    // it. It is already an absolute path since we resolved it to one above.
+    build.onResolve({ filter: /\.node$/, namespace: 'node-file' }, args => ({
+      path: args.path,
+      namespace: 'file',
+    }))
+
+    // Tell esbuild's default loading behavior to use the "file" loader for
+    // these ".node" files.
+    let opts = build.initialOptions
+    opts.loader = opts.loader || {}
+    opts.loader['.node'] = 'file'
+  },
+}
+
+
 const postCssPlugin: esbuild.Plugin = {
   name: "custom-build-postcss",
   setup(build) {