Movatterモバイル変換

chromium /chromium /src /refs/heads/main /. /build /locale_tool.py

blob: c9fd395b486834b85d4f9c21f65b22cff76be8ae [file] [log] [blame] [edit]

	#!/usr/bin/env vpython3
	# Copyright 2019 The Chromium Authors
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	"""Helper script used to manage locale-related files in Chromium.

	This script is used to check, and potentially fix, many locale-related files
	in your Chromium workspace, such as:

	- GRIT input files (.grd) and the corresponding translations (.xtb).

	- BUILD.gn files listing Android localized resource string resource .xml
	generated by GRIT for all supported Chrome locales. These correspond to
	<output> elements that use the type="android" attribute.

	The --scan-dir <dir> option can be used to check for all files under a specific
	directory, and the --fix-inplace option can be used to try fixing any file
	that doesn't pass the check.

	This can be very handy to avoid tedious and repetitive work when adding new
	translations / locales to the Chrome code base, since this script can update
	said input files for you.

	Important note: checks and fix may fail on some input files. For example
	remoting/resources/remoting_strings.grd contains an in-line comment element
	inside its <outputs> section that breaks the script. The check will fail, and
	trying to fix it too, but at least the file will not be modified.
	"""


	import argparse
	import json
	import os
	import re
	import shutil
	import subprocess
	import sys
	import unittest

	# Assume this script is under build/
	_SCRIPT_DIR= os.path.dirname(__file__)
	_SCRIPT_NAME= os.path.join(_SCRIPT_DIR, os.path.basename(__file__))
	_TOP_SRC_DIR= os.path.join(_SCRIPT_DIR,'..')

	# Need to import android/gyp/util/resource_utils.py here.
	sys.path.insert(0, os.path.join(_SCRIPT_DIR,'android/gyp'))

	from utilimport build_utils
	from utilimport resource_utils


	# This locale is the default and doesn't have translations.
	_DEFAULT_LOCALE='en-US'

	# Misc terminal codes to provide human friendly progress output.
	_CONSOLE_CODE_MOVE_CURSOR_TO_COLUMN_0='\x1b[0G'
	_CONSOLE_CODE_ERASE_LINE='\x1b[K'
	_CONSOLE_START_LINE=(
	_CONSOLE_CODE_MOVE_CURSOR_TO_COLUMN_0+ _CONSOLE_CODE_ERASE_LINE)

	##########################################################################
	##########################################################################
	#####
	##### G E N E R I C H E L P E R F U N C T I O N S
	#####
	##########################################################################
	##########################################################################

	def_FixChromiumLangAttribute(lang):
	"""Map XML "lang" attribute values to Chromium locale names."""
	_CHROMIUM_LANG_FIXES={
	'en':'en-US',# For now, Chromium doesn't have an 'en' locale.
	'iw':'he',# 'iw' is the obsolete form of ISO 639-1 for Hebrew
	'no':'nb',# 'no' is used by the Translation Console for Norwegian (nb).
	}
	return _CHROMIUM_LANG_FIXES.get(lang, lang)


	def_FixTranslationConsoleLocaleName(locale):
	_FIXES={
	'nb':'no',# Norwegian.
	'he':'iw',# Hebrew
	}
	return _FIXES.get(locale, locale)


	def_CompareLocaleLists(list_a, list_expected, list_name):
	"""Compare two lists of locale names. Print errors if they differ.

	Args:
	list_a: First list of locales.
	list_expected: Second list of locales, as expected.
	list_name: Name of list printed in error messages.
	Returns:
	On success, return False. On error, print error messages and return True.
	"""
	errors=[]
	missing_locales= sorted(set(list_a)- set(list_expected))
	if missing_locales:
	errors.append('Missing locales: %s'% missing_locales)

	extra_locales= sorted(set(list_expected)- set(list_a))
	if extra_locales:
	errors.append('Unexpected locales: %s'% extra_locales)

	if errors:
	print('Errors in %s definition:'% list_name)
	for errorin errors:
	print(' %s\n'% error)
	returnTrue

	returnFalse


	def_BuildIntervalList(input_list, predicate):
	"""Find ranges of contiguous list items that pass a given predicate.

	Args:
	input_list: An input list of items of any type.
	predicate: A function that takes a list item and return True if it
	passes a given test.
	Returns:
	A list of (start_pos, end_pos) tuples, where all items in
	[start_pos, end_pos) pass the predicate.
	"""
	result=[]
	size= len(input_list)
	start=0
	whileTrue:
	# Find first item in list that passes the predicate.
	while start< sizeandnot predicate(input_list[start]):
	start+=1

	if start>= size:
	return result

	# Find first item in the rest of the list that does not pass the
	# predicate.
	end= start+1
	while end< sizeand predicate(input_list[end]):
	end+=1

	result.append((start, end))
	start= end+1


	def_SortListSubRange(input_list, start, end, key_func):
	"""Sort an input list's sub-range according to a specific key function.

	Args:
	input_list: An input list.
	start: Sub-range starting position in list.
	end: Sub-range limit position in list.
	key_func: A function that extracts a sort key from a line.
	Returns:
	A copy of \|input_list\|, with all items in [\|start\|, \|end\|) sorted
	according to \|key_func\|.
	"""
	result= input_list[:start]
	inputs=[]
	for posin xrange(start, end):
	line= input_list[pos]
	key= key_func(line)
	inputs.append((key, line))

	for _, linein sorted(inputs):
	result.append(line)

	result+= input_list[end:]
	return result


	def_SortElementsRanges(lines, element_predicate, element_key):
	"""Sort all elements of a given type in a list of lines by a given key.

	Args:
	lines: input lines.
	element_predicate: predicate function to select elements to sort.
	element_key: lambda returning a comparison key for each element that
	passes the predicate.
	Returns:
	A new list of input lines, with lines [start..end) sorted.
	"""
	intervals=_BuildIntervalList(lines, element_predicate)
	for start, endin intervals:
	lines=_SortListSubRange(lines, start, end, element_key)

	return lines


	def_ProcessFile(input_file, locales, check_func, fix_func):
	"""Process a given input file, potentially fixing it.

	Args:
	input_file: Input file path.
	locales: List of Chrome locales to consider / expect.
	check_func: A lambda called to check the input file lines with
	(input_lines, locales) argument. It must return an list of error
	messages, or None on success.
	fix_func: None, or a lambda called to fix the input file lines with
	(input_lines, locales). It must return the new list of lines for
	the input file, and may raise an Exception in case of error.
	Returns:
	True at the moment.
	"""
	print('%sProcessing %s...'%(_CONSOLE_START_LINE, input_file), end=' ')
	sys.stdout.flush()
	with open(input_file)as f:
	input_lines= f.readlines()
	errors= check_func(input_file, input_lines, locales)
	if errors:
	print('\n%s%s'%(_CONSOLE_START_LINE,'\n'.join(errors)))
	if fix_func:
	try:
	input_lines= fix_func(input_file, input_lines, locales)
	output=''.join(input_lines)
	with open(input_file,'wt')as f:
	f.write(output)
	print('Fixed %s.'% input_file)
	exceptExceptionas e:# pylint: disable=broad-except
	print('Skipped %s: %s'%(input_file, e))

	returnTrue


	def_ScanDirectoriesForFiles(scan_dirs, file_predicate):
	"""Scan a directory for files that match a given predicate.

	Args:
	scan_dir: A list of top-level directories to start scan in.
	file_predicate: lambda function which is passed the file's base name
	and returns True if its full path, relative to \|scan_dir\|, should be
	passed in the result.
	Returns:
	A list of file full paths.
	"""
	result=[]
	for src_dirin scan_dirs:
	for root, _, filesin os.walk(src_dir):
	result.extend(os.path.join(root, f)for fin filesif file_predicate(f))
	return result


	def_WriteFile(file_path, file_data):
	"""Write \|file_data\| to \|file_path\|."""
	with open(file_path,'w')as f:
	f.write(file_data)


	def_FindGnExecutable():
	"""Locate the real GN executable used by this Chromium checkout.

	This is needed because the depot_tools 'gn' wrapper script will look
	for .gclient and other things we really don't need here.

	Returns:
	Path of real host GN executable from current Chromium src/ checkout.
	"""
	# Simply scan buildtools/*/gn and return the first one found so we don't
	# have to guess the platform-specific sub-directory name (e.g. 'linux64'
	# for 64-bit Linux machines).
	buildtools_dir= os.path.join(_TOP_SRC_DIR,'buildtools')
	for subdirin os.listdir(buildtools_dir):
	subdir_path= os.path.join(buildtools_dir, subdir)
	ifnot os.path.isdir(subdir_path):
	continue
	gn_path= os.path.join(subdir_path,'gn')
	if os.path.exists(gn_path):
	return gn_path
	returnNone


	def_PrettyPrintListAsLines(input_list, available_width, trailing_comma=False):
	result=[]
	input_str=', '.join(input_list)
	while len(input_str)> available_width:
	pos= input_str.rfind(',',0, available_width)
	result.append(input_str[:pos+1])
	input_str= input_str[pos+1:].lstrip()
	if trailing_commaand input_str:
	input_str+=','
	result.append(input_str)
	return result


	class_PrettyPrintListAsLinesTest(unittest.TestCase):

	def test_empty_list(self):
	self.assertListEqual([''],_PrettyPrintListAsLines([],10))

	def test_wrapping(self):
	input_list=['foo','bar','zoo','tool']
	self.assertListEqual(
	_PrettyPrintListAsLines(input_list,8),
	['foo,','bar,','zoo,','tool'])
	self.assertListEqual(
	_PrettyPrintListAsLines(input_list,12),['foo, bar,','zoo, tool'])
	self.assertListEqual(
	_PrettyPrintListAsLines(input_list,79),['foo, bar, zoo, tool'])

	def test_trailing_comma(self):
	input_list=['foo','bar','zoo','tool']
	self.assertListEqual(
	_PrettyPrintListAsLines(input_list,8, trailing_comma=True),
	['foo,','bar,','zoo,','tool,'])
	self.assertListEqual(
	_PrettyPrintListAsLines(input_list,12, trailing_comma=True),
	['foo, bar,','zoo, tool,'])
	self.assertListEqual(
	_PrettyPrintListAsLines(input_list,79, trailing_comma=True),
	['foo, bar, zoo, tool,'])


	##########################################################################
	##########################################################################
	#####
	##### L O C A L E S L I S T S
	#####
	##########################################################################
	##########################################################################

	# Various list of locales that will be extracted from build/config/locales.gni
	# Do not use these directly, use ChromeLocales(), and IosUnsupportedLocales()
	# instead to access these lists.
	_INTERNAL_CHROME_LOCALES=[]
	_INTERNAL_IOS_UNSUPPORTED_LOCALES=[]


	defChromeLocales():
	"""Return the list of all locales supported by Chrome."""
	ifnot _INTERNAL_CHROME_LOCALES:
	_ExtractAllChromeLocalesLists()
	return _INTERNAL_CHROME_LOCALES


	defIosUnsupportedLocales():
	"""Return the list of locales that are unsupported on iOS."""
	ifnot _INTERNAL_IOS_UNSUPPORTED_LOCALES:
	_ExtractAllChromeLocalesLists()
	return _INTERNAL_IOS_UNSUPPORTED_LOCALES


	def_PrepareTinyGnWorkspace(work_dir, out_subdir_name='out'):
	"""Populate an empty directory with a tiny set of working GN config files.

	This allows us to run 'gn gen <out> --root <work_dir>' as fast as possible
	to generate files containing the locales list. This takes about 300ms on
	a decent machine, instead of more than 5 seconds when running the equivalent
	commands from a real Chromium workspace, which requires regenerating more
	than 23k targets.

	Args:
	work_dir: target working directory.
	out_subdir_name: Name of output sub-directory.
	Returns:
	Full path of output directory created inside \|work_dir\|.
	"""
	# Create top-level .gn file that must point to the BUILDCONFIG.gn.
	_WriteFile(os.path.join(work_dir,'.gn'),
	'buildconfig = "//BUILDCONFIG.gn"\n')
	# Create BUILDCONFIG.gn which must set a default toolchain. Also add
	# all variables that may be used in locales.gni in a declare_args() block.
	_WriteFile(
	os.path.join(work_dir,'BUILDCONFIG.gn'),
	r'''set_default_toolchain("toolchain")
	declare_args () {
	is_ios = false
	is_android = true
	}
	''')

	# Create fake toolchain required by BUILDCONFIG.gn.
	os.mkdir(os.path.join(work_dir,'toolchain'))
	_WriteFile(os.path.join(work_dir,'toolchain','BUILD.gn'),
	r'''toolchain("toolchain") {
	tool("stamp") {
	command = "touch {{output}}" # Required by action()
	}
	}
	''')

	# Create top-level BUILD.gn, GN requires at least one target to build so do
	# that with a fake action which will never be invoked. Also write the locales
	# to misc files in the output directory.
	_WriteFile(
	os.path.join(work_dir,'BUILD.gn'), r'''import("//locales.gni")

	action("create_foo") { # fake action to avoid GN complaints.
	script = "//build/create_foo.py"
	inputs = []
	outputs = [ "$target_out_dir/$target_name" ]
	}

	# Write the locales lists to files in the output directory.
	_filename = root_build_dir + "/foo"
	write_file(_filename + ".locales", locales, "json")
	write_file(_filename + ".ios_unsupported_locales",
	ios_unsupported_locales,
	"json")
	''')

	# Copy build/config/locales.gni to the workspace, as required by BUILD.gn.
	shutil.copyfile(os.path.join(_TOP_SRC_DIR,'build','config','locales.gni'),
	os.path.join(work_dir,'locales.gni'))

	# Create output directory.
	out_path= os.path.join(work_dir, out_subdir_name)
	os.mkdir(out_path)

	# And ... we're good.
	return out_path


	# Set this global variable to the path of a given temporary directory
	# before calling _ExtractAllChromeLocalesLists() if you want to debug
	# the locales list extraction process.
	_DEBUG_LOCALES_WORK_DIR=None


	def_ReadJsonList(file_path):
	"""Read a JSON file that must contain a list, and return it."""
	with open(file_path)as f:
	data= json.load(f)
	assert isinstance(data, list),"JSON file %s is not a list!"% file_path
	return[item.encode('utf8')for itemin data]


	def_ExtractAllChromeLocalesLists():
	with build_utils.TempDir()as tmp_path:
	if _DEBUG_LOCALES_WORK_DIR:
	tmp_path= _DEBUG_LOCALES_WORK_DIR
	build_utils.DeleteDirectory(tmp_path)
	build_utils.MakeDirectory(tmp_path)

	out_path=_PrepareTinyGnWorkspace(tmp_path,'out')

	# NOTE: The file suffixes used here should be kept in sync with
	# build/config/locales.gni
	gn_executable=_FindGnExecutable()
	try:
	subprocess.check_output(
	[gn_executable,'gen', out_path,'--root='+ tmp_path])
	except subprocess.CalledProcessErroras e:
	print(e.output)
	raise e

	global _INTERNAL_CHROME_LOCALES
	_INTERNAL_CHROME_LOCALES=_ReadJsonList(
	os.path.join(out_path,'foo.locales'))

	global _INTERNAL_IOS_UNSUPPORTED_LOCALES
	_INTERNAL_IOS_UNSUPPORTED_LOCALES=_ReadJsonList(
	os.path.join(out_path,'foo.ios_unsupported_locales'))


	##########################################################################
	##########################################################################
	#####
	##### G R D H E L P E R F U N C T I O N S
	#####
	##########################################################################
	##########################################################################

	# Technical note:
	#
	# Even though .grd files are XML, an xml parser library is not used in order
	# to preserve the original file's structure after modification. ElementTree
	# tends to re-order attributes in each element when re-writing an XML
	# document tree, which is undesirable here.
	#
	# Thus simple line-based regular expression matching is used instead.
	#

	# Misc regular expressions used to match elements and their attributes.
	_RE_OUTPUT_ELEMENT= re.compile(r'<output (.)\s/>')
	_RE_TRANSLATION_ELEMENT= re.compile(r'<file( \| .* )path="(.\.xtb)"./>')
	_RE_FILENAME_ATTRIBUTE= re.compile(r'filename="([^"]*)"')
	_RE_LANG_ATTRIBUTE= re.compile(r'lang="([^"]*)"')
	_RE_PATH_ATTRIBUTE= re.compile(r'path="([^"]*)"')
	_RE_TYPE_ANDROID_ATTRIBUTE= re.compile(r'type="android"')



	def_IsGritInputFile(input_file):
	"""Returns True iff this is a GRIT input file."""
	return input_file.endswith('.grd')


	def_GetXmlLangAttribute(xml_line):
	"""Extract the lang attribute value from an XML input line."""
	m= _RE_LANG_ATTRIBUTE.search(xml_line)
	ifnot m:
	returnNone
	return m.group(1)


	class_GetXmlLangAttributeTest(unittest.TestCase):
	TEST_DATA={
	'':None,
	'foo':None,
	'lang=foo':None,
	'lang="foo"':'foo',
	'<something lang="foo bar" />':'foo bar',
	'<file lang="fr-CA" path="path/to/strings_fr-CA.xtb" />':'fr-CA',
	}

	def test_GetXmlLangAttribute(self):
	for test_line, expectedin self.TEST_DATA.items():
	self.assertEquals(_GetXmlLangAttribute(test_line), expected)


	def_SortGrdElementsRanges(grd_lines, element_predicate):
	"""Sort all .grd elements of a given type by their lang attribute."""
	return_SortElementsRanges(grd_lines, element_predicate,_GetXmlLangAttribute)


	def_CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales):
	"""Check the element 'lang' attributes in specific .grd lines range.

	This really checks the following:
	- Each item has a correct 'lang' attribute.
	- There are no duplicated lines for the same 'lang' attribute.
	- That there are no extra locales that Chromium doesn't want.
	- That no wanted locale is missing.

	Args:
	grd_lines: Input .grd lines.
	start: Sub-range start position in input line list.
	end: Sub-range limit position in input line list.
	wanted_locales: Set of wanted Chromium locale names.
	Returns:
	List of error message strings for this input. Empty on success.
	"""
	errors=[]
	locales= set()
	for posin xrange(start, end):
	line= grd_lines[pos]
	lang=_GetXmlLangAttribute(line)
	ifnot lang:
	errors.append('%d: Missing "lang" attribute in <output> element'% pos+
	1)
	continue
	cr_locale=_FixChromiumLangAttribute(lang)
	if cr_localein locales:
	errors.append(
	'%d: Redefinition of <output> for "%s" locale'%(pos+1, lang))
	locales.add(cr_locale)

	extra_locales= locales.difference(wanted_locales)
	if extra_locales:
	errors.append('%d-%d: Extra locales found: %s'%(start+1, end+1,
	sorted(extra_locales)))

	missing_locales= wanted_locales.difference(locales)
	if missing_locales:
	errors.append('%d-%d: Missing locales: %s'%(start+1, end+1,
	sorted(missing_locales)))

	return errors


	##########################################################################
	##########################################################################
	#####
	##### G R D A N D R O I D O U T P U T S
	#####
	##########################################################################
	##########################################################################

	def_IsGrdAndroidOutputLine(line):
	"""Returns True iff this is an Android-specific <output> line."""
	m= _RE_OUTPUT_ELEMENT.search(line)
	if m:
	return'type="android"'in m.group(1)
	returnFalse

	assert_IsGrdAndroidOutputLine(' <output type="android"/>')

	# Many of the functions below have unused arguments due to genericity.
	# pylint: disable=unused-argument

	def_CheckGrdElementRangeAndroidOutputFilename(grd_lines, start, end,
	wanted_locales):
	"""Check all <output> elements in specific input .grd lines range.

	This really checks the following:
	- Filenames exist for each listed locale.
	- Filenames are well-formed.

	Args:
	grd_lines: Input .grd lines.
	start: Sub-range start position in input line list.
	end: Sub-range limit position in input line list.
	wanted_locales: Set of wanted Chromium locale names.
	Returns:
	List of error message strings for this input. Empty on success.
	"""
	errors=[]
	for posin xrange(start, end):
	line= grd_lines[pos]
	lang=_GetXmlLangAttribute(line)
	ifnot lang:
	continue
	cr_locale=_FixChromiumLangAttribute(lang)

	m= _RE_FILENAME_ATTRIBUTE.search(line)
	ifnot m:
	errors.append('%d: Missing filename attribute in <output> element'% pos+
	1)
	else:
	filename= m.group(1)
	ifnot filename.endswith('.xml'):
	errors.append(
	'%d: Filename should end with ".xml": %s'%(pos+1, filename))

	dirname= os.path.basename(os.path.dirname(filename))
	prefix=('values-%s'% resource_utils.ToAndroidLocaleName(cr_locale)
	if cr_locale!= _DEFAULT_LOCALEelse'values')
	if dirname!= prefix:
	errors.append(
	'%s: Directory name should be %s: %s'%(pos+1, prefix, filename))

	return errors


	def_CheckGrdAndroidOutputElements(grd_file, grd_lines, wanted_locales):
	"""Check all <output> elements related to Android.

	Args:
	grd_file: Input .grd file path.
	grd_lines: List of input .grd lines.
	wanted_locales: set of wanted Chromium locale names.
	Returns:
	List of error message strings. Empty on success.
	"""
	intervals=_BuildIntervalList(grd_lines,_IsGrdAndroidOutputLine)
	errors=[]
	for start, endin intervals:
	errors+=_CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales)
	errors+=_CheckGrdElementRangeAndroidOutputFilename(grd_lines, start, end,
	wanted_locales)
	return errors


	def_AddMissingLocalesInGrdAndroidOutputs(grd_file, grd_lines, wanted_locales):
	"""Fix an input .grd line by adding missing Android outputs.

	Args:
	grd_file: Input .grd file path.
	grd_lines: Input .grd line list.
	wanted_locales: set of Chromium locale names.
	Returns:
	A new list of .grd lines, containing new <output> elements when needed
	for locales from \|wanted_locales\| that were not part of the input.
	"""
	intervals=_BuildIntervalList(grd_lines,_IsGrdAndroidOutputLine)
	for start, endin reversed(intervals):
	locales= set()
	for posin xrange(start, end):
	lang=_GetXmlLangAttribute(grd_lines[pos])
	locale=_FixChromiumLangAttribute(lang)
	locales.add(locale)

	missing_locales= wanted_locales.difference(locales)
	ifnot missing_locales:
	continue

	src_locale='bg'
	src_lang_attribute='lang="%s"'% src_locale
	src_line=None
	for posin xrange(start, end):
	if src_lang_attributein grd_lines[pos]:
	src_line= grd_lines[pos]
	break

	ifnot src_line:
	raiseException(
	'Cannot find <output> element with "%s" lang attribute'% src_locale)

	line_count= end-1
	for localein missing_locales:
	android_locale= resource_utils.ToAndroidLocaleName(locale)
	dst_line= src_line.replace(
	'lang="%s"'% src_locale,'lang="%s"'% locale).replace(
	'values-%s/'% src_locale,'values-%s/'% android_locale)
	grd_lines.insert(line_count, dst_line)
	line_count+=1

	# Sort the new <output> elements.
	return_SortGrdElementsRanges(grd_lines,_IsGrdAndroidOutputLine)


	##########################################################################
	##########################################################################
	#####
	##### G R D T R A N S L A T I O N S
	#####
	##########################################################################
	##########################################################################


	def_IsTranslationGrdOutputLine(line):
	"""Returns True iff this is an output .xtb <file> element."""
	m= _RE_TRANSLATION_ELEMENT.search(line)
	return misnotNone


	class_IsTranslationGrdOutputLineTest(unittest.TestCase):

	def test_GrdTranslationOutputLines(self):
	_VALID_INPUT_LINES=[
	'<file path="foo/bar.xtb" />',
	'<file path="foo/bar.xtb"/>',
	'<file lang="fr-CA" path="translations/aw_strings_fr-CA.xtb"/>',
	'<file lang="fr-CA" path="translations/aw_strings_fr-CA.xtb" />',
	' <file path="translations/aw_strings_ar.xtb" lang="ar" />',
	]
	_INVALID_INPUT_LINES=['<file path="foo/bar.xml" />']

	for linein _VALID_INPUT_LINES:
	self.assertTrue(
	_IsTranslationGrdOutputLine(line),
	'_IsTranslationGrdOutputLine() returned False for [%s]'% line)

	for linein _INVALID_INPUT_LINES:
	self.assertFalse(
	_IsTranslationGrdOutputLine(line),
	'_IsTranslationGrdOutputLine() returned True for [%s]'% line)


	def_CheckGrdTranslationElementRange(grd_lines, start, end,
	wanted_locales):
	"""Check all <translations> sub-elements in specific input .grd lines range.

	This really checks the following:
	- Each item has a 'path' attribute.
	- Each such path value ends up with '.xtb'.

	Args:
	grd_lines: Input .grd lines.
	start: Sub-range start position in input line list.
	end: Sub-range limit position in input line list.
	wanted_locales: Set of wanted Chromium locale names.
	Returns:
	List of error message strings for this input. Empty on success.
	"""
	errors=[]
	for posin xrange(start, end):
	line= grd_lines[pos]
	lang=_GetXmlLangAttribute(line)
	ifnot lang:
	continue
	m= _RE_PATH_ATTRIBUTE.search(line)
	ifnot m:
	errors.append('%d: Missing path attribute in <file> element'% pos+
	1)
	else:
	filename= m.group(1)
	ifnot filename.endswith('.xtb'):
	errors.append(
	'%d: Path should end with ".xtb": %s'%(pos+1, filename))

	return errors


	def_CheckGrdTranslations(grd_file, grd_lines, wanted_locales):
	"""Check all <file> elements that correspond to an .xtb output file.

	Args:
	grd_file: Input .grd file path.
	grd_lines: List of input .grd lines.
	wanted_locales: set of wanted Chromium locale names.
	Returns:
	List of error message strings. Empty on success.
	"""
	wanted_locales= wanted_locales- set([_DEFAULT_LOCALE])
	intervals=_BuildIntervalList(grd_lines,_IsTranslationGrdOutputLine)
	errors=[]
	for start, endin intervals:
	errors+=_CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales)
	errors+=_CheckGrdTranslationElementRange(grd_lines, start, end,
	wanted_locales)
	return errors


	# Regular expression used to replace the lang attribute inside .xtb files.
	_RE_TRANSLATIONBUNDLE= re.compile('<translationbundle lang="(.*)">')


	def_CreateFakeXtbFileFrom(src_xtb_path, dst_xtb_path, dst_locale):
	"""Create a fake .xtb file.

	Args:
	src_xtb_path: Path to source .xtb file to copy from.
	dst_xtb_path: Path to destination .xtb file to write to.
	dst_locale: Destination locale, the lang attribute in the source file
	will be substituted with this value before its lines are written
	to the destination file.
	"""
	with open(src_xtb_path)as f:
	src_xtb_lines= f.readlines()

	def replace_xtb_lang_attribute(line):
	m= _RE_TRANSLATIONBUNDLE.search(line)
	ifnot m:
	return line
	return line[:m.start(1)]+ dst_locale+ line[m.end(1):]

	dst_xtb_lines=[replace_xtb_lang_attribute(line)for linein src_xtb_lines]
	with build_utils.AtomicOutput(dst_xtb_path)as tmp:
	tmp.writelines(dst_xtb_lines)


	def_AddMissingLocalesInGrdTranslations(grd_file, grd_lines, wanted_locales):
	"""Fix an input .grd line by adding missing Android outputs.

	This also creates fake .xtb files from the one provided for 'en-GB'.

	Args:
	grd_file: Input .grd file path.
	grd_lines: Input .grd line list.
	wanted_locales: set of Chromium locale names.
	Returns:
	A new list of .grd lines, containing new <output> elements when needed
	for locales from \|wanted_locales\| that were not part of the input.
	"""
	wanted_locales= wanted_locales- set([_DEFAULT_LOCALE])
	intervals=_BuildIntervalList(grd_lines,_IsTranslationGrdOutputLine)
	for start, endin reversed(intervals):
	locales= set()
	for posin xrange(start, end):
	lang=_GetXmlLangAttribute(grd_lines[pos])
	locale=_FixChromiumLangAttribute(lang)
	locales.add(locale)

	missing_locales= wanted_locales.difference(locales)
	ifnot missing_locales:
	continue

	src_locale='en-GB'
	src_lang_attribute='lang="%s"'% src_locale
	src_line=None
	for posin xrange(start, end):
	if src_lang_attributein grd_lines[pos]:
	src_line= grd_lines[pos]
	break

	ifnot src_line:
	raiseException(
	'Cannot find <file> element with "%s" lang attribute'% src_locale)

	src_path= os.path.join(
	os.path.dirname(grd_file),
	_RE_PATH_ATTRIBUTE.search(src_line).group(1))

	line_count= end-1
	for localein missing_locales:
	dst_line= src_line.replace(
	'lang="%s"'% src_locale,'lang="%s"'% locale).replace(
	'_%s.xtb'% src_locale,'_%s.xtb'% locale)
	grd_lines.insert(line_count, dst_line)
	line_count+=1

	dst_path= src_path.replace('_%s.xtb'% src_locale,'_%s.xtb'% locale)
	_CreateFakeXtbFileFrom(src_path, dst_path, locale)


	# Sort the new <output> elements.
	return_SortGrdElementsRanges(grd_lines,_IsTranslationGrdOutputLine)


	##########################################################################
	##########################################################################
	#####
	##### G N A N D R O I D O U T P U T S
	#####
	##########################################################################
	##########################################################################

	_RE_GN_VALUES_LIST_LINE= re.compile(
	r'^\s".values(\-([A-Za-z0-9-]+))?/.\.xml",\s$')

	def_IsBuildGnInputFile(input_file):
	"""Returns True iff this is a BUILD.gn file."""
	return os.path.basename(input_file)=='BUILD.gn'


	def_GetAndroidGnOutputLocale(line):
	"""Check a GN list, and return its Android locale if it is an output .xml"""
	m= _RE_GN_VALUES_LIST_LINE.match(line)
	ifnot m:
	returnNone

	if m.group(1):# First group is optional and contains group 2.
	return m.group(2)

	return resource_utils.ToAndroidLocaleName(_DEFAULT_LOCALE)


	def_IsAndroidGnOutputLine(line):
	"""Returns True iff this is an Android-specific localized .xml output."""
	return_GetAndroidGnOutputLocale(line)!=None


	def_CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
	"""Check that a range of GN lines corresponds to localized strings.

	Special case: Some BUILD.gn files list several non-localized .xml files
	that should be ignored by this function, e.g. in
	components/cronet/android/BUILD.gn, the following appears:

	inputs = [
	...
	"sample/res/layout/activity_main.xml",
	"sample/res/layout/dialog_url.xml",
	"sample/res/values/dimens.xml",
	"sample/res/values/strings.xml",
	...
	]

	These are non-localized strings, and should be ignored. This function is
	used to detect them quickly.
	"""
	for posin xrange(start, end):
	ifnot'values/'in gn_lines[pos]:
	returnTrue
	returnFalse


	def_CheckGnOutputsRange(gn_lines, start, end, wanted_locales):
	ifnot_CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
	return[]

	errors=[]
	locales= set()
	for posin xrange(start, end):
	line= gn_lines[pos]
	android_locale=_GetAndroidGnOutputLocale(line)
	assert android_locale!=None
	cr_locale= resource_utils.ToChromiumLocaleName(android_locale)
	if cr_localein locales:
	errors.append('%s: Redefinition of output for "%s" locale'%
	(pos+1, android_locale))
	locales.add(cr_locale)

	extra_locales= locales.difference(wanted_locales)
	if extra_locales:
	errors.append('%d-%d: Extra locales: %s'%(start+1, end+1,
	sorted(extra_locales)))

	missing_locales= wanted_locales.difference(locales)
	if missing_locales:
	errors.append('%d-%d: Missing locales: %s'%(start+1, end+1,
	sorted(missing_locales)))

	return errors


	def_CheckGnAndroidOutputs(gn_file, gn_lines, wanted_locales):
	intervals=_BuildIntervalList(gn_lines,_IsAndroidGnOutputLine)
	errors=[]
	for start, endin intervals:
	errors+=_CheckGnOutputsRange(gn_lines, start, end, wanted_locales)
	return errors


	def_AddMissingLocalesInGnAndroidOutputs(gn_file, gn_lines, wanted_locales):
	intervals=_BuildIntervalList(gn_lines,_IsAndroidGnOutputLine)
	# NOTE: Since this may insert new lines to each interval, process the
	# list in reverse order to maintain valid (start,end) positions during
	# the iteration.
	for start, endin reversed(intervals):
	ifnot_CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
	continue

	locales= set()
	for posin xrange(start, end):
	lang=_GetAndroidGnOutputLocale(gn_lines[pos])
	locale= resource_utils.ToChromiumLocaleName(lang)
	locales.add(locale)

	missing_locales= wanted_locales.difference(locales)
	ifnot missing_locales:
	continue

	src_locale='bg'
	src_values='values-%s/'% resource_utils.ToAndroidLocaleName(src_locale)
	src_line=None
	for posin xrange(start, end):
	if src_valuesin gn_lines[pos]:
	src_line= gn_lines[pos]
	break

	ifnot src_line:
	raiseException(
	'Cannot find output list item with "%s" locale'% src_locale)

	line_count= end-1
	for localein missing_locales:
	if locale== _DEFAULT_LOCALE:
	dst_line= src_line.replace('values-%s/'% src_locale,'values/')
	else:
	dst_line= src_line.replace(
	'values-%s/'% src_locale,
	'values-%s/'% resource_utils.ToAndroidLocaleName(locale))
	gn_lines.insert(line_count, dst_line)
	line_count+=1

	gn_lines=_SortListSubRange(
	gn_lines, start, line_count,
	lambda line: _RE_GN_VALUES_LIST_LINE.match(line).group(1))

	return gn_lines


	##########################################################################
	##########################################################################
	#####
	##### T R A N S L A T I O N E X P E C T A T I O N S
	#####
	##########################################################################
	##########################################################################

	_EXPECTATIONS_FILENAME='translation_expectations.pyl'

	# Technical note: the format of translation_expectations.pyl
	# is a 'Python literal', which defines a python dictionary, so should
	# be easy to parse. However, when modifying it, care should be taken
	# to respect the line comments and the order of keys within the text
	# file.


	def_ReadPythonLiteralFile(pyl_path):
	"""Read a .pyl file into a Python data structure."""
	with open(pyl_path)as f:
	pyl_content= f.read()
	# Evaluate as a Python data structure, use an empty global
	# and local dictionary.
	return eval(pyl_content, dict(), dict())


	def_UpdateLocalesInExpectationLines(pyl_lines,
	wanted_locales,
	available_width=79):
	"""Update the locales list(s) found in an expectations file.

	Args:
	pyl_lines: Iterable of input lines from the file.
	wanted_locales: Set or list of new locale names.
	available_width: Optional, number of character colums used
	to word-wrap the new list items.
	Returns:
	New list of updated lines.
	"""
	locales_list=['"%s"'% locfor locin sorted(wanted_locales)]
	result=[]
	line_count= len(pyl_lines)
	line_num=0
	DICT_START='"languages": ['
	while line_num< line_count:
	line= pyl_lines[line_num]
	line_num+=1
	result.append(line)
	# Look for start of "languages" dictionary.
	pos= line.find(DICT_START)
	if pos<0:
	continue

	start_margin= pos
	start_line= line_num
	# Skip over all lines from the list.
	while(line_num< line_countand
	not pyl_lines[line_num].rstrip().endswith('],')):
	line_num+=1
	continue

	if line_num== line_count:
	raiseException('%d: Missing list termination!'% start_line)

	# Format the new list according to the new margin.
	locale_width= available_width-(start_margin+2)
	locale_lines=_PrettyPrintListAsLines(
	locales_list, locale_width, trailing_comma=True)
	for locale_linein locale_lines:
	result.append(' '*(start_margin+2)+ locale_line)
	result.append(' '* start_margin+'],')
	line_num+=1

	return result


	class_UpdateLocalesInExpectationLinesTest(unittest.TestCase):

	def test_simple(self):
	self.maxDiff=1000
	input_text= r'''
	# This comment should be preserved
	# 23456789012345678901234567890123456789
	{
	"android_grd": {
	"languages": [
	"aa", "bb", "cc", "dd", "ee",
	"ff", "gg", "hh", "ii", "jj",
	"kk"],
	},
	# Example with bad indentation in input.
	"another_grd": {
	"languages": [
	"aa", "bb", "cc", "dd", "ee", "ff", "gg", "hh", "ii", "jj", "kk",
	],
	},
	}
	'''
	expected_text= r'''
	# This comment should be preserved
	# 23456789012345678901234567890123456789
	{
	"android_grd": {
	"languages": [
	"A2", "AA", "BB", "CC", "DD",
	"E2", "EE", "FF", "GG", "HH",
	"I2", "II", "JJ", "KK",
	],
	},
	# Example with bad indentation in input.
	"another_grd": {
	"languages": [
	"A2", "AA", "BB", "CC", "DD",
	"E2", "EE", "FF", "GG", "HH",
	"I2", "II", "JJ", "KK",
	],
	},
	}
	'''
	input_lines= input_text.splitlines()
	test_locales=([
	'AA','BB','CC','DD','EE','FF','GG','HH','II','JJ','KK','A2',
	'E2','I2'
	])
	expected_lines= expected_text.splitlines()
	self.assertListEqual(
	_UpdateLocalesInExpectationLines(input_lines, test_locales,40),
	expected_lines)

	def test_missing_list_termination(self):
	input_lines= r'''
	"languages": ['
	"aa", "bb", "cc", "dd"
	'''.splitlines()
	with self.assertRaises(Exception)as cm:
	_UpdateLocalesInExpectationLines(input_lines,['a','b'],40)

	self.assertEqual(str(cm.exception),'2: Missing list termination!')


	def_UpdateLocalesInExpectationFile(pyl_path, wanted_locales):
	"""Update all locales listed in a given expectations file.

	Args:
	pyl_path: Path to .pyl file to update.
	wanted_locales: List of locales that need to be written to
	the file.
	"""
	tc_locales={
	_FixTranslationConsoleLocaleName(locale)
	for localein set(wanted_locales)- set([_DEFAULT_LOCALE])
	}

	with open(pyl_path)as f:
	input_lines=[l.rstrip()for lin f.readlines()]

	updated_lines=_UpdateLocalesInExpectationLines(input_lines, tc_locales)
	with build_utils.AtomicOutput(pyl_path)as f:
	f.writelines('\n'.join(updated_lines)+'\n')


	##########################################################################
	##########################################################################
	#####
	##### C H E C K E V E R Y T H I N G
	#####
	##########################################################################
	##########################################################################

	# pylint: enable=unused-argument


	def_IsAllInputFile(input_file):
	return_IsGritInputFile(input_file)or_IsBuildGnInputFile(input_file)


	def_CheckAllFiles(input_file, input_lines, wanted_locales):
	errors=[]
	if_IsGritInputFile(input_file):
	errors+=_CheckGrdTranslations(input_file, input_lines, wanted_locales)
	errors+=_CheckGrdAndroidOutputElements(
	input_file, input_lines, wanted_locales)
	elif_IsBuildGnInputFile(input_file):
	errors+=_CheckGnAndroidOutputs(input_file, input_lines, wanted_locales)
	return errors


	def_AddMissingLocalesInAllFiles(input_file, input_lines, wanted_locales):
	if_IsGritInputFile(input_file):
	lines=_AddMissingLocalesInGrdTranslations(
	input_file, input_lines, wanted_locales)
	lines=_AddMissingLocalesInGrdAndroidOutputs(
	input_file, lines, wanted_locales)
	elif_IsBuildGnInputFile(input_file):
	lines=_AddMissingLocalesInGnAndroidOutputs(
	input_file, input_lines, wanted_locales)
	return lines


	##########################################################################
	##########################################################################
	#####
	##### C O M M A N D H A N D L I N G
	#####
	##########################################################################
	##########################################################################

	class_Command(object):
	"""A base class for all commands recognized by this script.

	Usage is the following:
	1) Derived classes must re-define the following class-based fields:
	- name: Command name (e.g. 'list-locales')
	- description: Command short description.
	- long_description: Optional. Command long description.
	NOTE: As a convenience, if the first character is a newline,
	it will be omitted in the help output.

	2) Derived classes for commands that take arguments should override
	RegisterExtraArgs(), which receives a corresponding argparse
	sub-parser as argument.

	3) Derived classes should implement a Run() command, which can read
	the current arguments from self.args.
	"""
	name=None
	description=None
	long_description=None

	def __init__(self):
	self._parser=None
	self.args=None

	defRegisterExtraArgs(self, subparser):
	pass

	defRegisterArgs(self, parser):
	subp= parser.add_parser(
	self.name, help=self.description,
	description=self.long_descriptionor self.description,
	formatter_class=argparse.RawDescriptionHelpFormatter)
	self._parser= subp
	subp.set_defaults(command=self)
	group= subp.add_argument_group('%s arguments'% self.name)
	self.RegisterExtraArgs(group)

	defProcessArgs(self, args):
	self.args= args


	class_ListLocalesCommand(_Command):
	"""Implement the 'list-locales' command to list locale lists of interest."""
	name='list-locales'
	description='List supported Chrome locales'
	long_description= r'''
	List locales of interest, by default this prints all locales supported by
	Chrome, but `--type=ios_unsupported` can be used for the list of locales
	unsupported on iOS.

	These values are extracted directly from build/config/locales.gni.

	Additionally, use the --as-json argument to print the list as a JSON list,
	instead of the default format (which is a space-separated list of locale names).
	'''

	# Maps type argument to a function returning the corresponding locales list.
	TYPE_MAP={
	'all':ChromeLocales,
	'ios_unsupported':IosUnsupportedLocales,
	}

	defRegisterExtraArgs(self, group):
	group.add_argument(
	'--as-json',
	action='store_true',
	help='Output as JSON list.')
	group.add_argument(
	'--type',
	choices=tuple(self.TYPE_MAP.viewkeys()),
	default='all',
	help='Select type of locale list to print.')

	defRun(self):
	locale_list= self.TYPE_MAP[self.args.type]()
	if self.args.as_json:
	print('[%s]'%", ".join("'%s'"% locfor locin locale_list))
	else:
	print(' '.join(locale_list))


	class_CheckInputFileBaseCommand(_Command):
	"""Used as a base for other _Command subclasses that check input files.

	Subclasses should also define the following class-level variables:

	- select_file_func:
	A predicate that receives a file name (not path) and return True if it
	should be selected for inspection. Used when scanning directories with
	'--scan-dir <dir>'.

	- check_func:
	- fix_func:
	Two functions passed as parameters to _ProcessFile(), see relevant
	documentation in this function's definition.
	"""
	select_file_func=None
	check_func=None
	fix_func=None

	defRegisterExtraArgs(self, group):
	group.add_argument(
	'--scan-dir',
	action='append',
	help='Optional directory to scan for input files recursively.')
	group.add_argument(
	'input',
	nargs='*',
	help='Input file(s) to check.')
	group.add_argument(
	'--fix-inplace',
	action='store_true',
	help='Try to fix the files in-place too.')
	group.add_argument(
	'--add-locales',
	help='Space-separated list of additional locales to use')

	defRun(self):
	args= self.args
	input_files=[]
	if args.input:
	input_files= args.input
	if args.scan_dir:
	input_files.extend(_ScanDirectoriesForFiles(
	args.scan_dir, self.select_file_func.__func__))
	locales=ChromeLocales()
	if args.add_locales:
	locales.extend(args.add_locales.split(' '))

	locales= set(locales)

	for input_filein input_files:
	_ProcessFile(input_file,
	locales,
	self.check_func.__func__,
	self.fix_func.__func__if args.fix_inplaceelseNone)
	print('%sDone.'%(_CONSOLE_START_LINE))


	class_CheckGrdAndroidOutputsCommand(_CheckInputFileBaseCommand):
	name='check-grd-android-outputs'
	description=(
	'Check the Android resource (.xml) files outputs in GRIT input files.')
	long_description= r'''
	Check the Android .xml files outputs in one or more input GRIT (.grd) files
	for the following conditions:

	- Each item has a correct 'lang' attribute.
	- There are no duplicated lines for the same 'lang' attribute.
	- That there are no extra locales that Chromium doesn't want.
	- That no wanted locale is missing.
	- Filenames exist for each listed locale.
	- Filenames are well-formed.
	'''
	select_file_func=_IsGritInputFile
	check_func=_CheckGrdAndroidOutputElements
	fix_func=_AddMissingLocalesInGrdAndroidOutputs


	class_CheckGrdTranslationsCommand(_CheckInputFileBaseCommand):
	name='check-grd-translations'
	description=(
	'Check the translation (.xtb) files outputted by .grd input files.')
	long_description= r'''
	Check the translation (.xtb) file outputs in one or more input GRIT (.grd) files
	for the following conditions:

	- Each item has a correct 'lang' attribute.
	- There are no duplicated lines for the same 'lang' attribute.
	- That there are no extra locales that Chromium doesn't want.
	- That no wanted locale is missing.
	- Each item has a 'path' attribute.
	- Each such path value ends up with '.xtb'.
	'''
	select_file_func=_IsGritInputFile
	check_func=_CheckGrdTranslations
	fix_func=_AddMissingLocalesInGrdTranslations


	class_CheckGnAndroidOutputsCommand(_CheckInputFileBaseCommand):
	name='check-gn-android-outputs'
	description='Check the Android .xml file lists in GN build files.'
	long_description= r'''
	Check one or more BUILD.gn file, looking for lists of Android resource .xml
	files, and checking that:

	- There are no duplicated output files in the list.
	- Each output file belongs to a wanted Chromium locale.
	- There are no output files for unwanted Chromium locales.
	'''
	select_file_func=_IsBuildGnInputFile
	check_func=_CheckGnAndroidOutputs
	fix_func=_AddMissingLocalesInGnAndroidOutputs


	class_CheckAllCommand(_CheckInputFileBaseCommand):
	name='check-all'
	description='Check everything.'
	long_description='Equivalent to calling all other check-xxx commands.'
	select_file_func=_IsAllInputFile
	check_func=_CheckAllFiles
	fix_func=_AddMissingLocalesInAllFiles


	class_UpdateExpectationsCommand(_Command):
	name='update-expectations'
	description='Update translation expectations file.'
	long_description= r'''
	Update %s files to match the current list of locales supported by Chromium.
	This is especially useful to add new locales before updating any GRIT or GN
	input file with the --add-locales option.
	'''% _EXPECTATIONS_FILENAME

	defRegisterExtraArgs(self, group):
	group.add_argument(
	'--add-locales',
	help='Space-separated list of additional locales to use.')

	defRun(self):
	locales=ChromeLocales()
	add_locales= self.args.add_locales
	if add_locales:
	locales.extend(add_locales.split(' '))

	expectation_paths=[
	'tools/gritsettings/translation_expectations.pyl',
	'clank/tools/translation_expectations.pyl',
	]
	missing_expectation_files=[]
	for pathin enumerate(expectation_paths):
	file_path= os.path.join(_TOP_SRC_DIR, path)
	ifnot os.path.exists(file_path):
	missing_expectation_files.append(file_path)
	continue
	_UpdateLocalesInExpectationFile(file_path, locales)

	if missing_expectation_files:
	sys.stderr.write('WARNING: Missing file(s): %s\n'%
	(', '.join(missing_expectation_files)))


	class_UnitTestsCommand(_Command):
	name='unit-tests'
	description='Run internal unit-tests for this script'

	defRegisterExtraArgs(self, group):
	group.add_argument(
	'-v','--verbose', action='count', help='Increase test verbosity.')
	group.add_argument('args', nargs=argparse.REMAINDER)

	defRun(self):
	argv=[_SCRIPT_NAME]+ self.args.args
	unittest.main(argv=argv, verbosity=self.args.verbose)


	# List of all commands supported by this script.
	_COMMANDS=[
	_ListLocalesCommand,
	_CheckGrdAndroidOutputsCommand,
	_CheckGrdTranslationsCommand,
	_CheckGnAndroidOutputsCommand,
	_CheckAllCommand,
	_UpdateExpectationsCommand,
	_UnitTestsCommand,
	]


	def main(argv):
	parser= argparse.ArgumentParser(
	description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)

	subparsers= parser.add_subparsers()
	commands=[clazz()for clazzin _COMMANDS]
	for commandin commands:
	command.RegisterArgs(subparsers)

	ifnot argv:
	argv=['--help']

	args= parser.parse_args(argv)
	args.command.ProcessArgs(args)
	args.command.Run()


	if __name__=="__main__":
	main(sys.argv[1:])