diff options
Diffstat (limited to 'deps/v8/build/locale_tool.py')
| -rwxr-xr-x | deps/v8/build/locale_tool.py | 1483 |
1 files changed, 1483 insertions, 0 deletions
diff --git a/deps/v8/build/locale_tool.py b/deps/v8/build/locale_tool.py new file mode 100755 index 0000000000..04d5cd3591 --- /dev/null +++ b/deps/v8/build/locale_tool.py @@ -0,0 +1,1483 @@ +#!/usr/bin/env vpython +# Copyright 2019 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Helper script used to manage locale-related files in Chromium. + +This script is used to check, and potentially fix, many locale-related files +in your Chromium workspace, such as: + + - GRIT input files (.grd) and the corresponding translations (.xtb). + + - BUILD.gn files listing Android localized resource string resource .xml + generated by GRIT for all supported Chrome locales. These correspond to + <output> elements that use the type="android" attribute. + +The --scan-dir <dir> option can be used to check for all files under a specific +directory, and the --fix-inplace option can be used to try fixing any file +that doesn't pass the check. + +This can be very handy to avoid tedious and repetitive work when adding new +translations / locales to the Chrome code base, since this script can update +said input files for you. + +Important note: checks and fix may fail on some input files. For example +remoting/resources/remoting_strings.grd contains an in-line comment element +inside its <outputs> section that breaks the script. The check will fail, and +trying to fix it too, but at least the file will not be modified. +""" +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import unittest + +# Assume this script is under build/ +_SCRIPT_DIR = os.path.dirname(__file__) +_SCRIPT_NAME = os.path.join(_SCRIPT_DIR, os.path.basename(__file__)) +_TOP_SRC_DIR = os.path.join(_SCRIPT_DIR, '..') + +# Need to import android/gyp/util/resource_utils.py here. +sys.path.insert(0, os.path.join(_SCRIPT_DIR, 'android/gyp')) + +from util import build_utils +from util import resource_utils + + +# This locale is the default and doesn't have translations. +_DEFAULT_LOCALE = 'en-US' + +# Misc terminal codes to provide human friendly progress output. +_CONSOLE_CODE_MOVE_CURSOR_TO_COLUMN_0 = '\x1b[0G' +_CONSOLE_CODE_ERASE_LINE = '\x1b[K' +_CONSOLE_START_LINE = ( + _CONSOLE_CODE_MOVE_CURSOR_TO_COLUMN_0 + _CONSOLE_CODE_ERASE_LINE) + +########################################################################## +########################################################################## +##### +##### G E N E R I C H E L P E R F U N C T I O N S +##### +########################################################################## +########################################################################## + +def _FixChromiumLangAttribute(lang): + """Map XML "lang" attribute values to Chromium locale names.""" + _CHROMIUM_LANG_FIXES = { + 'en': 'en-US', # For now, Chromium doesn't have an 'en' locale. + 'iw': 'he', # 'iw' is the obsolete form of ISO 639-1 for Hebrew + 'no': 'nb', # 'no' is used by the Translation Console for Norwegian (nb). + } + return _CHROMIUM_LANG_FIXES.get(lang, lang) + + +def _FixTranslationConsoleLocaleName(locale): + _FIXES = { + 'nb': 'no', # Norwegian. + 'he': 'iw', # Hebrew + } + return _FIXES.get(locale, locale) + + +def _CompareLocaleLists(list_a, list_expected, list_name): + """Compare two lists of locale names. Print errors if they differ. + + Args: + list_a: First list of locales. + list_expected: Second list of locales, as expected. + list_name: Name of list printed in error messages. + Returns: + On success, return False. On error, print error messages and return True. + """ + errors = [] + missing_locales = sorted(set(list_a) - set(list_expected)) + if missing_locales: + errors.append('Missing locales: %s' % missing_locales) + + extra_locales = sorted(set(list_expected) - set(list_a)) + if extra_locales: + errors.append('Unexpected locales: %s' % extra_locales) + + if errors: + print 'Errors in %s definition:' % list_name + for error in errors: + print ' %s\n' % error + return True + + return False + + +def _BuildIntervalList(input_list, predicate): + """Find ranges of contiguous list items that pass a given predicate. + + Args: + input_list: An input list of items of any type. + predicate: A function that takes a list item and return True if it + passes a given test. + Returns: + A list of (start_pos, end_pos) tuples, where all items in + [start_pos, end_pos) pass the predicate. + """ + result = [] + size = len(input_list) + start = 0 + while True: + # Find first item in list that passes the predicate. + while start < size and not predicate(input_list[start]): + start += 1 + + if start >= size: + return result + + # Find first item in the rest of the list that does not pass the + # predicate. + end = start + 1 + while end < size and predicate(input_list[end]): + end += 1 + + result.append((start, end)) + start = end + 1 + + +def _SortListSubRange(input_list, start, end, key_func): + """Sort an input list's sub-range according to a specific key function. + + Args: + input_list: An input list. + start: Sub-range starting position in list. + end: Sub-range limit position in list. + key_func: A function that extracts a sort key from a line. + Returns: + A copy of |input_list|, with all items in [|start|, |end|) sorted + according to |key_func|. + """ + result = input_list[:start] + inputs = [] + for pos in xrange(start, end): + line = input_list[pos] + key = key_func(line) + inputs.append((key, line)) + + for _, line in sorted(inputs): + result.append(line) + + result += input_list[end:] + return result + + +def _SortElementsRanges(lines, element_predicate, element_key): + """Sort all elements of a given type in a list of lines by a given key. + + Args: + lines: input lines. + element_predicate: predicate function to select elements to sort. + element_key: lambda returning a comparison key for each element that + passes the predicate. + Returns: + A new list of input lines, with lines [start..end) sorted. + """ + intervals = _BuildIntervalList(lines, element_predicate) + for start, end in intervals: + lines = _SortListSubRange(lines, start, end, element_key) + + return lines + + +def _ProcessFile(input_file, locales, check_func, fix_func): + """Process a given input file, potentially fixing it. + + Args: + input_file: Input file path. + locales: List of Chrome locales to consider / expect. + check_func: A lambda called to check the input file lines with + (input_lines, locales) argument. It must return an list of error + messages, or None on success. + fix_func: None, or a lambda called to fix the input file lines with + (input_lines, locales). It must return the new list of lines for + the input file, and may raise an Exception in case of error. + Returns: + True at the moment. + """ + print '%sProcessing %s...' % (_CONSOLE_START_LINE, input_file), + sys.stdout.flush() + with open(input_file) as f: + input_lines = f.readlines() + errors = check_func(input_file, input_lines, locales) + if errors: + print '\n%s%s' % (_CONSOLE_START_LINE, '\n'.join(errors)) + if fix_func: + try: + input_lines = fix_func(input_file, input_lines, locales) + output = ''.join(input_lines) + with open(input_file, 'wt') as f: + f.write(output) + print 'Fixed %s.' % input_file + except Exception as e: # pylint: disable=broad-except + print 'Skipped %s: %s' % (input_file, e) + + return True + + +def _ScanDirectoriesForFiles(scan_dirs, file_predicate): + """Scan a directory for files that match a given predicate. + + Args: + scan_dir: A list of top-level directories to start scan in. + file_predicate: lambda function which is passed the file's base name + and returns True if its full path, relative to |scan_dir|, should be + passed in the result. + Returns: + A list of file full paths. + """ + result = [] + for src_dir in scan_dirs: + for root, _, files in os.walk(src_dir): + result.extend(os.path.join(root, f) for f in files if file_predicate(f)) + return result + + +def _WriteFile(file_path, file_data): + """Write |file_data| to |file_path|.""" + with open(file_path, 'w') as f: + f.write(file_data) + + +def _FindGnExecutable(): + """Locate the real GN executable used by this Chromium checkout. + + This is needed because the depot_tools 'gn' wrapper script will look + for .gclient and other things we really don't need here. + + Returns: + Path of real host GN executable from current Chromium src/ checkout. + """ + # Simply scan buildtools/*/gn and return the first one found so we don't + # have to guess the platform-specific sub-directory name (e.g. 'linux64' + # for 64-bit Linux machines). + buildtools_dir = os.path.join(_TOP_SRC_DIR, 'buildtools') + for subdir in os.listdir(buildtools_dir): + subdir_path = os.path.join(buildtools_dir, subdir) + if not os.path.isdir(subdir_path): + continue + gn_path = os.path.join(subdir_path, 'gn') + if os.path.exists(gn_path): + return gn_path + return None + + +def _PrettyPrintListAsLines(input_list, available_width, trailing_comma=False): + result = [] + input_str = ', '.join(input_list) + while len(input_str) > available_width: + pos = input_str.rfind(',', 0, available_width) + result.append(input_str[:pos + 1]) + input_str = input_str[pos + 1:].lstrip() + if trailing_comma and input_str: + input_str += ',' + result.append(input_str) + return result + + +class _PrettyPrintListAsLinesTest(unittest.TestCase): + + def test_empty_list(self): + self.assertListEqual([''], _PrettyPrintListAsLines([], 10)) + + def test_wrapping(self): + input_list = ['foo', 'bar', 'zoo', 'tool'] + self.assertListEqual( + _PrettyPrintListAsLines(input_list, 8), + ['foo,', 'bar,', 'zoo,', 'tool']) + self.assertListEqual( + _PrettyPrintListAsLines(input_list, 12), ['foo, bar,', 'zoo, tool']) + self.assertListEqual( + _PrettyPrintListAsLines(input_list, 79), ['foo, bar, zoo, tool']) + + def test_trailing_comma(self): + input_list = ['foo', 'bar', 'zoo', 'tool'] + self.assertListEqual( + _PrettyPrintListAsLines(input_list, 8, trailing_comma=True), + ['foo,', 'bar,', 'zoo,', 'tool,']) + self.assertListEqual( + _PrettyPrintListAsLines(input_list, 12, trailing_comma=True), + ['foo, bar,', 'zoo, tool,']) + self.assertListEqual( + _PrettyPrintListAsLines(input_list, 79, trailing_comma=True), + ['foo, bar, zoo, tool,']) + + +########################################################################## +########################################################################## +##### +##### L O C A L E S L I S T S +##### +########################################################################## +########################################################################## + +# Various list of locales that will be extracted from build/config/locales.gni +# Do not use these directly, use ChromeLocales(), AndroidOmittedLocales() and +# IosUnsupportedLocales() instead to access these lists. +_INTERNAL_CHROME_LOCALES = [] +_INTERNAL_ANDROID_OMITTED_LOCALES = [] +_INTERNAL_IOS_UNSUPPORTED_LOCALES = [] + + +def ChromeLocales(): + """Return the list of all locales supported by Chrome.""" + if not _INTERNAL_CHROME_LOCALES: + _ExtractAllChromeLocalesLists() + return _INTERNAL_CHROME_LOCALES + + +def AndroidOmittedLocales(): + """Reutrn the list of locales omitted from Android APKs.""" + if not _INTERNAL_ANDROID_OMITTED_LOCALES: + _ExtractAllChromeLocalesLists() + return _INTERNAL_ANDROID_OMITTED_LOCALES + + +def IosUnsupportedLocales(): + """Return the list of locales that are unsupported on iOS.""" + if not _INTERNAL_IOS_UNSUPPORTED_LOCALES: + _ExtractAllChromeLocalesLists() + return _INTERNAL_IOS_UNSUPPORTED_LOCALES + + +def _PrepareTinyGnWorkspace(work_dir, out_subdir_name='out'): + """Populate an empty directory with a tiny set of working GN config files. + + This allows us to run 'gn gen <out> --root <work_dir>' as fast as possible + to generate files containing the locales list. This takes about 300ms on + a decent machine, instead of more than 5 seconds when running the equivalent + commands from a real Chromium workspace, which requires regenerating more + than 23k targets. + + Args: + work_dir: target working directory. + out_subdir_name: Name of output sub-directory. + Returns: + Full path of output directory created inside |work_dir|. + """ + # Create top-level .gn file that must point to the BUILDCONFIG.gn. + _WriteFile(os.path.join(work_dir, '.gn'), + 'buildconfig = "//BUILDCONFIG.gn"\n') + # Create BUILDCONFIG.gn which must set a default toolchain. Also add + # all variables that may be used in locales.gni in a declare_args() block. + _WriteFile(os.path.join(work_dir, 'BUILDCONFIG.gn'), + r'''set_default_toolchain("toolchain") +declare_args () { + is_ios = false +} +''') + + # Create fake toolchain required by BUILDCONFIG.gn. + os.mkdir(os.path.join(work_dir, 'toolchain')) + _WriteFile(os.path.join(work_dir, 'toolchain', 'BUILD.gn'), + r'''toolchain("toolchain") { + tool("stamp") { + command = "touch {{output}}" # Required by action() + } +} +''') + + # Create top-level BUILD.gn, GN requires at least one target to build so do + # that with a fake action which will never be invoked. Also write the locales + # to misc files in the output directory. + _WriteFile(os.path.join(work_dir, 'BUILD.gn'), + r'''import("//locales.gni") + +action("create_foo") { # fake action to avoid GN complaints. + script = "//build/create_foo.py" + inputs = [] + outputs = [ "$target_out_dir/$target_name" ] +} + +# Write the locales lists to files in the output directory. +_filename = root_build_dir + "/foo" +write_file(_filename + ".locales", locales, "json") +write_file(_filename + ".android_omitted_locales", + android_chrome_omitted_locales, + "json") +write_file(_filename + ".ios_unsupported_locales", + ios_unsupported_locales, + "json") +''') + + # Copy build/config/locales.gni to the workspace, as required by BUILD.gn. + shutil.copyfile(os.path.join(_TOP_SRC_DIR, 'build', 'config', 'locales.gni'), + os.path.join(work_dir, 'locales.gni')) + + # Create output directory. + out_path = os.path.join(work_dir, out_subdir_name) + os.mkdir(out_path) + + # And ... we're good. + return out_path + + +# Set this global variable to the path of a given temporary directory +# before calling _ExtractAllChromeLocalesLists() if you want to debug +# the locales list extraction process. +_DEBUG_LOCALES_WORK_DIR = None + + +def _ReadJsonList(file_path): + """Read a JSON file that must contain a list, and return it.""" + with open(file_path) as f: + data = json.load(f) + assert isinstance(data, list), "JSON file %s is not a list!" % file_path + return [item.encode('utf8') for item in data] + + +def _ExtractAllChromeLocalesLists(): + with build_utils.TempDir() as tmp_path: + if _DEBUG_LOCALES_WORK_DIR: + tmp_path = _DEBUG_LOCALES_WORK_DIR + build_utils.DeleteDirectory(tmp_path) + build_utils.MakeDirectory(tmp_path) + + out_path = _PrepareTinyGnWorkspace(tmp_path, 'out') + + # NOTE: The file suffixes used here should be kept in sync with + # build/config/locales.gni + gn_executable = _FindGnExecutable() + subprocess.check_output( + [gn_executable, 'gen', out_path, '--root=' + tmp_path]) + + global _INTERNAL_CHROME_LOCALES + _INTERNAL_CHROME_LOCALES = _ReadJsonList( + os.path.join(out_path, 'foo.locales')) + + global _INTERNAL_ANDROID_OMITTED_LOCALES + _INTERNAL_ANDROID_OMITTED_LOCALES = _ReadJsonList( + os.path.join(out_path, 'foo.android_omitted_locales')) + + global _INTERNAL_IOS_UNSUPPORTED_LOCALES + _INTERNAL_IOS_UNSUPPORTED_LOCALES = _ReadJsonList( + os.path.join(out_path, 'foo.ios_unsupported_locales')) + + +########################################################################## +########################################################################## +##### +##### G R D H E L P E R F U N C T I O N S +##### +########################################################################## +########################################################################## + +# Technical note: +# +# Even though .grd files are XML, an xml parser library is not used in order +# to preserve the original file's structure after modification. ElementTree +# tends to re-order attributes in each element when re-writing an XML +# document tree, which is undesirable here. +# +# Thus simple line-based regular expression matching is used instead. +# + +# Misc regular expressions used to match elements and their attributes. +_RE_OUTPUT_ELEMENT = re.compile(r'<output (.*)\s*/>') +_RE_TRANSLATION_ELEMENT = re.compile(r'<file (.*\.xtb")\s*/>') +_RE_FILENAME_ATTRIBUTE = re.compile(r'filename="([^"]*)"') +_RE_LANG_ATTRIBUTE = re.compile(r'lang="([^"]*)"') +_RE_PATH_ATTRIBUTE = re.compile(r'path="([^"]*)"') +_RE_TYPE_ANDROID_ATTRIBUTE = re.compile(r'type="android"') + +assert _RE_TRANSLATION_ELEMENT.match('<file path="foo/bar.xtb" />') +assert _RE_TRANSLATION_ELEMENT.match('<file path="foo/bar.xtb"/>') +assert _RE_TRANSLATION_ELEMENT.match('<file path="foo/bar.xml" />') is None + + +def _IsGritInputFile(input_file): + """Returns True iff this is a GRIT input file.""" + return input_file.endswith('.grd') + + +def _SortGrdElementsRanges(grd_lines, element_predicate): + """Sort all .grd elements of a given type by their lang attribute.""" + return _SortElementsRanges( + grd_lines, + element_predicate, + lambda x: _RE_LANG_ATTRIBUTE.search(x).group(1)) + + +def _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales): + """Check the element 'lang' attributes in specific .grd lines range. + + This really checks the following: + - Each item has a correct 'lang' attribute. + - There are no duplicated lines for the same 'lang' attribute. + - That there are no extra locales that Chromium doesn't want. + - That no wanted locale is missing. + + Args: + grd_lines: Input .grd lines. + start: Sub-range start position in input line list. + end: Sub-range limit position in input line list. + wanted_locales: Set of wanted Chromium locale names. + Returns: + List of error message strings for this input. Empty on success. + """ + errors = [] + locales = set() + for pos in xrange(start, end): + line = grd_lines[pos] + m = _RE_LANG_ATTRIBUTE.search(line) + if not m: + errors.append('%d: Missing "lang" attribute in <output> element' % pos + + 1) + continue + lang = m.group(1) + cr_locale = _FixChromiumLangAttribute(lang) + if cr_locale in locales: + errors.append( + '%d: Redefinition of <output> for "%s" locale' % (pos + 1, lang)) + locales.add(cr_locale) + + extra_locales = locales.difference(wanted_locales) + if extra_locales: + errors.append('%d-%d: Extra locales found: %s' % (start + 1, end + 1, + sorted(extra_locales))) + + missing_locales = wanted_locales.difference(locales) + if missing_locales: + errors.append('%d-%d: Missing locales: %s' % (start + 1, end + 1, + sorted(missing_locales))) + + return errors + + +########################################################################## +########################################################################## +##### +##### G R D A N D R O I D O U T P U T S +##### +########################################################################## +########################################################################## + +def _IsGrdAndroidOutputLine(line): + """Returns True iff this is an Android-specific <output> line.""" + m = _RE_OUTPUT_ELEMENT.search(line) + if m: + return 'type="android"' in m.group(1) + return False + +assert _IsGrdAndroidOutputLine(' <output type="android"/>') + +# Many of the functions below have unused arguments due to genericity. +# pylint: disable=unused-argument + +def _CheckGrdElementRangeAndroidOutputFilename(grd_lines, start, end, + wanted_locales): + """Check all <output> elements in specific input .grd lines range. + + This really checks the following: + - Filenames exist for each listed locale. + - Filenames are well-formed. + + Args: + grd_lines: Input .grd lines. + start: Sub-range start position in input line list. + end: Sub-range limit position in input line list. + wanted_locales: Set of wanted Chromium locale names. + Returns: + List of error message strings for this input. Empty on success. + """ + errors = [] + for pos in xrange(start, end): + line = grd_lines[pos] + m = _RE_LANG_ATTRIBUTE.search(line) + if not m: + continue + lang = m.group(1) + cr_locale = _FixChromiumLangAttribute(lang) + + m = _RE_FILENAME_ATTRIBUTE.search(line) + if not m: + errors.append('%d: Missing filename attribute in <output> element' % pos + + 1) + else: + filename = m.group(1) + if not filename.endswith('.xml'): + errors.append( + '%d: Filename should end with ".xml": %s' % (pos + 1, filename)) + + dirname = os.path.basename(os.path.dirname(filename)) + prefix = ('values-%s' % resource_utils.ToAndroidLocaleName(cr_locale) + if cr_locale != _DEFAULT_LOCALE else 'values') + if dirname != prefix: + errors.append( + '%s: Directory name should be %s: %s' % (pos + 1, prefix, filename)) + + return errors + + +def _CheckGrdAndroidOutputElements(grd_file, grd_lines, wanted_locales): + """Check all <output> elements related to Android. + + Args: + grd_file: Input .grd file path. + grd_lines: List of input .grd lines. + wanted_locales: set of wanted Chromium locale names. + Returns: + List of error message strings. Empty on success. + """ + intervals = _BuildIntervalList(grd_lines, _IsGrdAndroidOutputLine) + errors = [] + for start, end in intervals: + errors += _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales) + errors += _CheckGrdElementRangeAndroidOutputFilename(grd_lines, start, end, + wanted_locales) + return errors + + +def _AddMissingLocalesInGrdAndroidOutputs(grd_file, grd_lines, wanted_locales): + """Fix an input .grd line by adding missing Android outputs. + + Args: + grd_file: Input .grd file path. + grd_lines: Input .grd line list. + wanted_locales: set of Chromium locale names. + Returns: + A new list of .grd lines, containing new <output> elements when needed + for locales from |wanted_locales| that were not part of the input. + """ + intervals = _BuildIntervalList(grd_lines, _IsGrdAndroidOutputLine) + for start, end in reversed(intervals): + locales = set() + for pos in xrange(start, end): + lang = _RE_LANG_ATTRIBUTE.search(grd_lines[pos]).group(1) + locale = _FixChromiumLangAttribute(lang) + locales.add(locale) + + missing_locales = wanted_locales.difference(locales) + if not missing_locales: + continue + + src_locale = 'bg' + src_lang_attribute = 'lang="%s"' % src_locale + src_line = None + for pos in xrange(start, end): + if src_lang_attribute in grd_lines[pos]: + src_line = grd_lines[pos] + break + + if not src_line: + raise Exception( + 'Cannot find <output> element with "%s" lang attribute' % src_locale) + + line_count = end - 1 + for locale in missing_locales: + android_locale = resource_utils.ToAndroidLocaleName(locale) + dst_line = src_line.replace( + 'lang="%s"' % src_locale, 'lang="%s"' % locale).replace( + 'values-%s/' % src_locale, 'values-%s/' % android_locale) + grd_lines.insert(line_count, dst_line) + line_count += 1 + + # Sort the new <output> elements. + return _SortGrdElementsRanges(grd_lines, _IsGrdAndroidOutputLine) + + +########################################################################## +########################################################################## +##### +##### G R D T R A N S L A T I O N S +##### +########################################################################## +########################################################################## + + +def _IsTranslationGrdOutputLine(line): + """Returns True iff this is an output .xtb <file> element.""" + m = _RE_TRANSLATION_ELEMENT.search(line) + return m is not None + + +def _CheckGrdTranslationElementRange(grd_lines, start, end, + wanted_locales): + """Check all <translations> sub-elements in specific input .grd lines range. + + This really checks the following: + - Each item has a 'path' attribute. + - Each such path value ends up with '.xtb'. + + Args: + grd_lines: Input .grd lines. + start: Sub-range start position in input line list. + end: Sub-range limit position in input line list. + wanted_locales: Set of wanted Chromium locale names. + Returns: + List of error message strings for this input. Empty on success. + """ + errors = [] + for pos in xrange(start, end): + line = grd_lines[pos] + m = _RE_LANG_ATTRIBUTE.search(line) + if not m: + continue + m = _RE_PATH_ATTRIBUTE.search(line) + if not m: + errors.append('%d: Missing path attribute in <file> element' % pos + + 1) + else: + filename = m.group(1) + if not filename.endswith('.xtb'): + errors.append( + '%d: Path should end with ".xtb": %s' % (pos + 1, filename)) + + return errors + + +def _CheckGrdTranslations(grd_file, grd_lines, wanted_locales): + """Check all <file> elements that correspond to an .xtb output file. + + Args: + grd_file: Input .grd file path. + grd_lines: List of input .grd lines. + wanted_locales: set of wanted Chromium locale names. + Returns: + List of error message strings. Empty on success. + """ + wanted_locales = wanted_locales - set([_DEFAULT_LOCALE]) + intervals = _BuildIntervalList(grd_lines, _IsTranslationGrdOutputLine) + errors = [] + for start, end in intervals: + errors += _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales) + errors += _CheckGrdTranslationElementRange(grd_lines, start, end, + wanted_locales) + return errors + + +# Regular expression used to replace the lang attribute inside .xtb files. +_RE_TRANSLATIONBUNDLE = re.compile('<translationbundle lang="(.*)">') + + +def _CreateFakeXtbFileFrom(src_xtb_path, dst_xtb_path, dst_locale): + """Create a fake .xtb file. + + Args: + src_xtb_path: Path to source .xtb file to copy from. + dst_xtb_path: Path to destination .xtb file to write to. + dst_locale: Destination locale, the lang attribute in the source file + will be substituted with this value before its lines are written + to the destination file. + """ + with open(src_xtb_path) as f: + src_xtb_lines = f.readlines() + + def replace_xtb_lang_attribute(line): + m = _RE_TRANSLATIONBUNDLE.search(line) + if not m: + return line + return line[:m.start(1)] + dst_locale + line[m.end(1):] + + dst_xtb_lines = [replace_xtb_lang_attribute(line) for line in src_xtb_lines] + with build_utils.AtomicOutput(dst_xtb_path) as tmp: + tmp.writelines(dst_xtb_lines) + + +def _AddMissingLocalesInGrdTranslations(grd_file, grd_lines, wanted_locales): + """Fix an input .grd line by adding missing Android outputs. + + This also creates fake .xtb files from the one provided for 'en-GB'. + + Args: + grd_file: Input .grd file path. + grd_lines: Input .grd line list. + wanted_locales: set of Chromium locale names. + Returns: + A new list of .grd lines, containing new <output> elements when needed + for locales from |wanted_locales| that were not part of the input. + """ + wanted_locales = wanted_locales - set([_DEFAULT_LOCALE]) + intervals = _BuildIntervalList(grd_lines, _IsTranslationGrdOutputLine) + for start, end in reversed(intervals): + locales = set() + for pos in xrange(start, end): + lang = _RE_LANG_ATTRIBUTE.search(grd_lines[pos]).group(1) + locale = _FixChromiumLangAttribute(lang) + locales.add(locale) + + missing_locales = wanted_locales.difference(locales) + if not missing_locales: + continue + + src_locale = 'en-GB' + src_lang_attribute = 'lang="%s"' % src_locale + src_line = None + for pos in xrange(start, end): + if src_lang_attribute in grd_lines[pos]: + src_line = grd_lines[pos] + break + + if not src_line: + raise Exception( + 'Cannot find <file> element with "%s" lang attribute' % src_locale) + + src_path = os.path.join( + os.path.dirname(grd_file), + _RE_PATH_ATTRIBUTE.search(src_line).group(1)) + + line_count = end - 1 + for locale in missing_locales: + dst_line = src_line.replace( + 'lang="%s"' % src_locale, 'lang="%s"' % locale).replace( + '_%s.xtb' % src_locale, '_%s.xtb' % locale) + grd_lines.insert(line_count, dst_line) + line_count += 1 + + dst_path = src_path.replace('_%s.xtb' % src_locale, '_%s.xtb' % locale) + _CreateFakeXtbFileFrom(src_path, dst_path, locale) + + + # Sort the new <output> elements. + return _SortGrdElementsRanges(grd_lines, _IsTranslationGrdOutputLine) + + +########################################################################## +########################################################################## +##### +##### G N A N D R O I D O U T P U T S +##### +########################################################################## +########################################################################## + +_RE_GN_VALUES_LIST_LINE = re.compile( + r'^\s*".*values(\-([A-Za-z0-9-]+))?/.*\.xml",\s*$') + +def _IsBuildGnInputFile(input_file): + """Returns True iff this is a BUILD.gn file.""" + return os.path.basename(input_file) == 'BUILD.gn' + + +def _GetAndroidGnOutputLocale(line): + """Check a GN list, and return its Android locale if it is an output .xml""" + m = _RE_GN_VALUES_LIST_LINE.match(line) + if not m: + return None + + if m.group(1): # First group is optional and contains group 2. + return m.group(2) + + return resource_utils.ToAndroidLocaleName(_DEFAULT_LOCALE) + + +def _IsAndroidGnOutputLine(line): + """Returns True iff this is an Android-specific localized .xml output.""" + return _GetAndroidGnOutputLocale(line) != None + + +def _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end): + """Check that a range of GN lines corresponds to localized strings. + + Special case: Some BUILD.gn files list several non-localized .xml files + that should be ignored by this function, e.g. in + components/cronet/android/BUILD.gn, the following appears: + + inputs = [ + ... + "sample/res/layout/activity_main.xml", + "sample/res/layout/dialog_url.xml", + "sample/res/values/dimens.xml", + "sample/res/values/strings.xml", + ... + ] + + These are non-localized strings, and should be ignored. This function is + used to detect them quickly. + """ + for pos in xrange(start, end): + if not 'values/' in gn_lines[pos]: + return True + return False + + +def _CheckGnOutputsRange(gn_lines, start, end, wanted_locales): + if not _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end): + return [] + + errors = [] + locales = set() + for pos in xrange(start, end): + line = gn_lines[pos] + android_locale = _GetAndroidGnOutputLocale(line) + assert android_locale != None + cr_locale = resource_utils.ToChromiumLocaleName(android_locale) + if cr_locale in locales: + errors.append('%s: Redefinition of output for "%s" locale' % + (pos + 1, android_locale)) + locales.add(cr_locale) + + extra_locales = locales.difference(wanted_locales) + if extra_locales: + errors.append('%d-%d: Extra locales: %s' % (start + 1, end + 1, + sorted(extra_locales))) + + missing_locales = wanted_locales.difference(locales) + if missing_locales: + errors.append('%d-%d: Missing locales: %s' % (start + 1, end + 1, + sorted(missing_locales))) + + return errors + + +def _CheckGnAndroidOutputs(gn_file, gn_lines, wanted_locales): + intervals = _BuildIntervalList(gn_lines, _IsAndroidGnOutputLine) + errors = [] + for start, end in intervals: + errors += _CheckGnOutputsRange(gn_lines, start, end, wanted_locales) + return errors + + +def _AddMissingLocalesInGnAndroidOutputs(gn_file, gn_lines, wanted_locales): + intervals = _BuildIntervalList(gn_lines, _IsAndroidGnOutputLine) + # NOTE: Since this may insert new lines to each interval, process the + # list in reverse order to maintain valid (start,end) positions during + # the iteration. + for start, end in reversed(intervals): + if not _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end): + continue + + locales = set() + for pos in xrange(start, end): + lang = _GetAndroidGnOutputLocale(gn_lines[pos]) + locale = resource_utils.ToChromiumLocaleName(lang) + locales.add(locale) + + missing_locales = wanted_locales.difference(locales) + if not missing_locales: + continue + + src_locale = 'bg' + src_values = 'values-%s/' % resource_utils.ToAndroidLocaleName(src_locale) + src_line = None + for pos in xrange(start, end): + if src_values in gn_lines[pos]: + src_line = gn_lines[pos] + break + + if not src_line: + raise Exception( + 'Cannot find output list item with "%s" locale' % src_locale) + + line_count = end - 1 + for locale in missing_locales: + if locale == _DEFAULT_LOCALE: + dst_line = src_line.replace('values-%s/' % src_locale, 'values/') + else: + dst_line = src_line.replace( + 'values-%s/' % src_locale, + 'values-%s/' % resource_utils.ToAndroidLocaleName(locale)) + gn_lines.insert(line_count, dst_line) + line_count += 1 + + gn_lines = _SortListSubRange( + gn_lines, start, line_count, + lambda line: _RE_GN_VALUES_LIST_LINE.match(line).group(1)) + + return gn_lines + + +########################################################################## +########################################################################## +##### +##### T R A N S L A T I O N E X P E C T A T I O N S +##### +########################################################################## +########################################################################## + +_EXPECTATIONS_FILENAME = 'translation_expectations.pyl' + +# Technical note: the format of translation_expectations.pyl +# is a 'Python literal', which defines a python dictionary, so should +# be easy to parse. However, when modifying it, care should be taken +# to respect the line comments and the order of keys within the text +# file. + + +def _ReadPythonLiteralFile(pyl_path): + """Read a .pyl file into a Python data structure.""" + with open(pyl_path) as f: + pyl_content = f.read() + # Evaluate as a Python data structure, use an empty global + # and local dictionary. + return eval(pyl_content, dict(), dict()) + + +def _UpdateLocalesInExpectationLines(pyl_lines, + wanted_locales, + available_width=79): + """Update the locales list(s) found in an expectations file. + + Args: + pyl_lines: Iterable of input lines from the file. + wanted_locales: Set or list of new locale names. + available_width: Optional, number of character colums used + to word-wrap the new list items. + Returns: + New list of updated lines. + """ + locales_list = ['"%s"' % loc for loc in sorted(wanted_locales)] + result = [] + line_count = len(pyl_lines) + line_num = 0 + DICT_START = '"languages": [' + while line_num < line_count: + line = pyl_lines[line_num] + line_num += 1 + result.append(line) + # Look for start of "languages" dictionary. + pos = line.find(DICT_START) + if pos < 0: + continue + + start_margin = pos + start_line = line_num + # Skip over all lines from the list. + while (line_num < line_count and + not pyl_lines[line_num].rstrip().endswith('],')): + line_num += 1 + continue + + if line_num == line_count: + raise Exception('%d: Missing list termination!' % start_line) + + # Format the new list according to the new margin. + locale_width = available_width - (start_margin + 2) + locale_lines = _PrettyPrintListAsLines( + locales_list, locale_width, trailing_comma=True) + for locale_line in locale_lines: + result.append(' ' * (start_margin + 2) + locale_line) + result.append(' ' * start_margin + '],') + line_num += 1 + + return result + + +class _UpdateLocalesInExpectationLinesTest(unittest.TestCase): + + def test_simple(self): + self.maxDiff = 1000 + input_text = r''' +# This comment should be preserved +# 23456789012345678901234567890123456789 +{ + "android_grd": { + "languages": [ + "aa", "bb", "cc", "dd", "ee", + "ff", "gg", "hh", "ii", "jj", + "kk"], + }, + # Example with bad indentation in input. + "another_grd": { + "languages": [ + "aa", "bb", "cc", "dd", "ee", "ff", "gg", "hh", "ii", "jj", "kk", + ], + }, +} +''' + expected_text = r''' +# This comment should be preserved +# 23456789012345678901234567890123456789 +{ + "android_grd": { + "languages": [ + "A2", "AA", "BB", "CC", "DD", + "E2", "EE", "FF", "GG", "HH", + "I2", "II", "JJ", "KK", + ], + }, + # Example with bad indentation in input. + "another_grd": { + "languages": [ + "A2", "AA", "BB", "CC", "DD", + "E2", "EE", "FF", "GG", "HH", + "I2", "II", "JJ", "KK", + ], + }, +} +''' + input_lines = input_text.splitlines() + test_locales = ([ + 'AA', 'BB', 'CC', 'DD', 'EE', 'FF', 'GG', 'HH', 'II', 'JJ', 'KK', 'A2', + 'E2', 'I2' + ]) + expected_lines = expected_text.splitlines() + self.assertListEqual( + _UpdateLocalesInExpectationLines(input_lines, test_locales, 40), + expected_lines) + + def test_missing_list_termination(self): + input_lines = r''' + "languages": [' + "aa", "bb", "cc", "dd" +'''.splitlines() + with self.assertRaises(Exception) as cm: + _UpdateLocalesInExpectationLines(input_lines, ['a', 'b'], 40) + + self.assertEqual(str(cm.exception), '2: Missing list termination!') + + +def _UpdateLocalesInExpectationFile(pyl_path, wanted_locales): + """Update all locales listed in a given expectations file. + + Args: + pyl_path: Path to .pyl file to update. + wanted_locales: List of locales that need to be written to + the file. + """ + tc_locales = { + _FixTranslationConsoleLocaleName(locale) + for locale in set(wanted_locales) - set([_DEFAULT_LOCALE]) + } + + with open(pyl_path) as f: + input_lines = [l.rstrip() for l in f.readlines()] + + updated_lines = _UpdateLocalesInExpectationLines(input_lines, tc_locales) + print repr(updated_lines) + with build_utils.AtomicOutput(pyl_path) as f: + f.writelines('\n'.join(updated_lines) + '\n') + + +########################################################################## +########################################################################## +##### +##### C H E C K E V E R Y T H I N G +##### +########################################################################## +########################################################################## + +# pylint: enable=unused-argument + + +def _IsAllInputFile(input_file): + return _IsGritInputFile(input_file) or _IsBuildGnInputFile(input_file) + + +def _CheckAllFiles(input_file, input_lines, wanted_locales): + errors = [] + if _IsGritInputFile(input_file): + errors += _CheckGrdTranslations(input_file, input_lines, wanted_locales) + errors += _CheckGrdAndroidOutputElements( + input_file, input_lines, wanted_locales) + elif _IsBuildGnInputFile(input_file): + errors += _CheckGnAndroidOutputs(input_file, input_lines, wanted_locales) + return errors + + +def _AddMissingLocalesInAllFiles(input_file, input_lines, wanted_locales): + if _IsGritInputFile(input_file): + lines = _AddMissingLocalesInGrdTranslations( + input_file, input_lines, wanted_locales) + lines = _AddMissingLocalesInGrdAndroidOutputs( + input_file, lines, wanted_locales) + elif _IsBuildGnInputFile(input_file): + lines = _AddMissingLocalesInGnAndroidOutputs( + input_file, input_lines, wanted_locales) + return lines + + +########################################################################## +########################################################################## +##### +##### C O M M A N D H A N D L I N G +##### +########################################################################## +########################################################################## + +class _Command(object): + """A base class for all commands recognized by this script. + + Usage is the following: + 1) Derived classes must re-define the following class-based fields: + - name: Command name (e.g. 'list-locales') + - description: Command short description. + - long_description: Optional. Command long description. + NOTE: As a convenience, if the first character is a newline, + it will be omitted in the help output. + + 2) Derived classes for commands that take arguments should override + RegisterExtraArgs(), which receives a corresponding argparse + sub-parser as argument. + + 3) Derived classes should implement a Run() command, which can read + the current arguments from self.args. + """ + name = None + description = None + long_description = None + + def __init__(self): + self._parser = None + self.args = None + + def RegisterExtraArgs(self, subparser): + pass + + def RegisterArgs(self, parser): + subp = parser.add_parser( + self.name, help=self.description, + description=self.long_description or self.description, + formatter_class=argparse.RawDescriptionHelpFormatter) + self._parser = subp + subp.set_defaults(command=self) + group = subp.add_argument_group('%s arguments' % self.name) + self.RegisterExtraArgs(group) + + def ProcessArgs(self, args): + self.args = args + + +class _ListLocalesCommand(_Command): + """Implement the 'list-locales' command to list locale lists of interest.""" + name = 'list-locales' + description = 'List supported Chrome locales' + long_description = r''' +List locales of interest, by default this prints all locales supported by +Chrome, but `--type=android_omitted` can be used to print the list of locales +omitted from Android APKs (but not app bundles), and `--type=ios_unsupported` +for the list of locales unsupported on iOS. + +These values are extracted directly from build/config/locales.gni. + +Additionally, use the --as-json argument to print the list as a JSON list, +instead of the default format (which is a space-separated list of locale names). +''' + + # Maps type argument to a function returning the corresponding locales list. + TYPE_MAP = { + 'all': ChromeLocales, + 'android_omitted': AndroidOmittedLocales, + 'ios_unsupported': IosUnsupportedLocales, + } + + def RegisterExtraArgs(self, group): + group.add_argument( + '--as-json', + action='store_true', + help='Output as JSON list.') + group.add_argument( + '--type', + choices=tuple(self.TYPE_MAP.viewkeys()), + default='all', + help='Select type of locale list to print.') + + def Run(self): + locale_list = self.TYPE_MAP[self.args.type]() + if self.args.as_json: + print '[%s]' % ", ".join("'%s'" % loc for loc in locale_list) + else: + print ' '.join(locale_list) + + +class _CheckInputFileBaseCommand(_Command): + """Used as a base for other _Command subclasses that check input files. + + Subclasses should also define the following class-level variables: + + - select_file_func: + A predicate that receives a file name (not path) and return True if it + should be selected for inspection. Used when scanning directories with + '--scan-dir <dir>'. + + - check_func: + - fix_func: + Two functions passed as parameters to _ProcessFile(), see relevant + documentation in this function's definition. + """ + select_file_func = None + check_func = None + fix_func = None + + def RegisterExtraArgs(self, group): + group.add_argument( + '--scan-dir', + action='append', + help='Optional directory to scan for input files recursively.') + group.add_argument( + 'input', + nargs='*', + help='Input file(s) to check.') + group.add_argument( + '--fix-inplace', + action='store_true', + help='Try to fix the files in-place too.') + group.add_argument( + '--add-locales', + help='Space-separated list of additional locales to use') + + def Run(self): + args = self.args + input_files = [] + if args.input: + input_files = args.input + if args.scan_dir: + input_files.extend(_ScanDirectoriesForFiles( + args.scan_dir, self.select_file_func.__func__)) + locales = ChromeLocales() + if args.add_locales: + locales.extend(args.add_locales.split(' ')) + + locales = set(locales) + + for input_file in input_files: + _ProcessFile(input_file, + locales, + self.check_func.__func__, + self.fix_func.__func__ if args.fix_inplace else None) + print '%sDone.' % (_CONSOLE_START_LINE) + + +class _CheckGrdAndroidOutputsCommand(_CheckInputFileBaseCommand): + name = 'check-grd-android-outputs' + description = ( + 'Check the Android resource (.xml) files outputs in GRIT input files.') + long_description = r''' +Check the Android .xml files outputs in one or more input GRIT (.grd) files +for the following conditions: + + - Each item has a correct 'lang' attribute. + - There are no duplicated lines for the same 'lang' attribute. + - That there are no extra locales that Chromium doesn't want. + - That no wanted locale is missing. + - Filenames exist for each listed locale. + - Filenames are well-formed. +''' + select_file_func = _IsGritInputFile + check_func = _CheckGrdAndroidOutputElements + fix_func = _AddMissingLocalesInGrdAndroidOutputs + + +class _CheckGrdTranslationsCommand(_CheckInputFileBaseCommand): + name = 'check-grd-translations' + description = ( + 'Check the translation (.xtb) files outputted by .grd input files.') + long_description = r''' +Check the translation (.xtb) file outputs in one or more input GRIT (.grd) files +for the following conditions: + + - Each item has a correct 'lang' attribute. + - There are no duplicated lines for the same 'lang' attribute. + - That there are no extra locales that Chromium doesn't want. + - That no wanted locale is missing. + - Each item has a 'path' attribute. + - Each such path value ends up with '.xtb'. +''' + select_file_func = _IsGritInputFile + check_func = _CheckGrdTranslations + fix_func = _AddMissingLocalesInGrdTranslations + + +class _CheckGnAndroidOutputsCommand(_CheckInputFileBaseCommand): + name = 'check-gn-android-outputs' + description = 'Check the Android .xml file lists in GN build files.' + long_description = r''' +Check one or more BUILD.gn file, looking for lists of Android resource .xml +files, and checking that: + + - There are no duplicated output files in the list. + - Each output file belongs to a wanted Chromium locale. + - There are no output files for unwanted Chromium locales. +''' + select_file_func = _IsBuildGnInputFile + check_func = _CheckGnAndroidOutputs + fix_func = _AddMissingLocalesInGnAndroidOutputs + + +class _CheckAllCommand(_CheckInputFileBaseCommand): + name = 'check-all' + description = 'Check everything.' + long_description = 'Equivalent to calling all other check-xxx commands.' + select_file_func = _IsAllInputFile + check_func = _CheckAllFiles + fix_func = _AddMissingLocalesInAllFiles + + +class _UpdateExpectationsCommand(_Command): + name = 'update-expectations' + description = 'Update translation expectations file.' + long_description = r''' +Update %s files to match the current list of locales supported by Chromium. +This is especially useful to add new locales before updating any GRIT or GN +input file with the --add-locales option. +''' % _EXPECTATIONS_FILENAME + + def RegisterExtraArgs(self, group): + group.add_argument( + '--add-locales', + help='Space-separated list of additional locales to use.') + + def Run(self): + locales = ChromeLocales() + add_locales = self.args.add_locales + if add_locales: + locales.extend(add_locales.split(' ')) + + expectation_paths = [ + 'tools/gritsettings/translation_expectations.pyl', + 'clank/tools/translation_expectations.pyl', + ] + missing_expectation_files = [] + for path in enumerate(expectation_paths): + file_path = os.path.join(_TOP_SRC_DIR, path) + if not os.path.exists(file_path): + missing_expectation_files.append(file_path) + continue + _UpdateLocalesInExpectationFile(file_path, locales) + + if missing_expectation_files: + sys.stderr.write('WARNING: Missing file(s): %s\n' % + (', '.join(missing_expectation_files))) + + +class _UnitTestsCommand(_Command): + name = 'unit-tests' + description = 'Run internal unit-tests for this script' + + def RegisterExtraArgs(self, group): + group.add_argument( + '-v', '--verbose', action='count', help='Increase test verbosity.') + group.add_argument('args', nargs=argparse.REMAINDER) + + def Run(self): + argv = [_SCRIPT_NAME] + self.args.args + unittest.main(argv=argv, verbosity=self.args.verbose) + + +# List of all commands supported by this script. +_COMMANDS = [ + _ListLocalesCommand, + _CheckGrdAndroidOutputsCommand, + _CheckGrdTranslationsCommand, + _CheckGnAndroidOutputsCommand, + _CheckAllCommand, + _UpdateExpectationsCommand, + _UnitTestsCommand, +] + + +def main(argv): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + + subparsers = parser.add_subparsers() + commands = [clazz() for clazz in _COMMANDS] + for command in commands: + command.RegisterArgs(subparsers) + + if not argv: + argv = ['--help'] + + args = parser.parse_args(argv) + args.command.ProcessArgs(args) + args.command.Run() + + +if __name__ == "__main__": + main(sys.argv[1:]) |