misc .arb fixes; localizations validator (#12197) * misc .arb fixes; localizations validator * regenerate localizations * address comments * do not treat plural variations as invalid keys
diff --git a/dev/bots/test.dart b/dev/bots/test.dart index a746a4d..c12ab27 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart
@@ -52,8 +52,42 @@ print('${bold}DONE: test.dart does nothing in the docs shard.$reset'); } +Future<Null> _verifyInternationalizations() async { + final EvalResult genResult = await _evalCommand( + dart, + <String>[ + path.join('dev', 'tools', 'gen_localizations.dart'), + path.join('packages', 'flutter', 'lib', 'src', 'material', 'i18n'), + 'material' + ], + workingDirectory: flutterRoot, + ); + + final String localizationsFile = path.join('packages', 'flutter', 'lib', 'src', 'material', 'i18n', 'localizations.dart'); + + final EvalResult sourceContents = await _evalCommand( + 'cat', + <String>[localizationsFile], + workingDirectory: flutterRoot, + ); + + if (genResult.stdout.trim() != sourceContents.stdout.trim()) { + stderr + ..writeln('<<<<<<< $localizationsFile') + ..writeln(sourceContents.stdout.trim()) + ..writeln('=======') + ..writeln(genResult.stdout.trim()) + ..writeln('>>>>>>> gen_localizations') + ..writeln('The contents of $localizationsFile are different from that produced by gen_localizations.') + ..writeln() + ..writeln('Did you forget to run gen_localizations.dart after updating a .arb file?'); + exit(1); + } +} + Future<Null> _analyzeRepo() async { await _verifyNoBadImports(flutterRoot); + await _verifyInternationalizations(); // Analyze all the Dart code in the repo. await _runFlutterAnalyze(flutterRoot, @@ -176,6 +210,57 @@ return _runCommand(pub, args, workingDirectory: workingDirectory); } +class EvalResult { + EvalResult({ + this.stdout, + this.stderr, + }); + + final String stdout; + final String stderr; +} + +Future<EvalResult> _evalCommand(String executable, List<String> arguments, { + String workingDirectory, + Map<String, String> environment, + bool skip: false, +}) async { + final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}'; + final String relativeWorkingDir = path.relative(workingDirectory); + if (skip) { + _printProgress('SKIPPING', relativeWorkingDir, commandDescription); + return null; + } + _printProgress('RUNNING', relativeWorkingDir, commandDescription); + + final Process process = await Process.start(executable, arguments, + workingDirectory: workingDirectory, + environment: environment, + ); + + final Future<List<List<int>>> savedStdout = process.stdout.toList(); + final Future<List<List<int>>> savedStderr = process.stderr.toList(); + final int exitCode = await process.exitCode; + final EvalResult result = new EvalResult( + stdout: UTF8.decode((await savedStdout).expand((List<int> ints) => ints).toList()), + stderr: UTF8.decode((await savedStderr).expand((List<int> ints) => ints).toList()), + ); + + if (exitCode != 0) { + stderr.write(result.stderr); + print( + '$red━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$reset\n' + '${bold}ERROR:$red Last command exited with $exitCode.$reset\n' + '${bold}Command:$red $commandDescription$reset\n' + '${bold}Relative working directory:$red $relativeWorkingDir$reset\n' + '$red━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$reset' + ); + exit(1); + } + + return result; +} + Future<Null> _runCommand(String executable, List<String> arguments, { String workingDirectory, Map<String, String> environment,
diff --git a/dev/tools/gen_localizations.dart b/dev/tools/gen_localizations.dart index 493044b..b3e9e61 100644 --- a/dev/tools/gen_localizations.dart +++ b/dev/tools/gen_localizations.dart
@@ -26,6 +26,8 @@ import 'dart:convert' show JSON; import 'dart:io'; +import 'localizations_validator.dart'; + const String outputHeader = ''' // Copyright 2017 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be @@ -36,8 +38,14 @@ // @(regenerate) '''; +/// Maps locales to resource key/value pairs. final Map<String, Map<String, String>> localeToResources = <String, Map<String, String>>{}; +/// Maps locales to resource attributes. +/// +/// See also https://github.com/googlei18n/app-resource-bundle/wiki/ApplicationResourceBundleSpecification#resource-attributes +final Map<String, Map<String, dynamic>> localeToResourceAttributes = <String, Map<String, dynamic>>{}; + // Return s as a Dart-parseable raw string in double quotes. Expand double quotes: // foo => r"foo" // foo "bar" => r"foo " '"' r"bar" '"' @@ -92,13 +100,16 @@ void processBundle(File file, String locale) { localeToResources[locale] ??= <String, String>{}; + localeToResourceAttributes[locale] ??= <String, dynamic>{}; final Map<String, String> resources = localeToResources[locale]; + final Map<String, dynamic> attributes = localeToResourceAttributes[locale]; final Map<String, dynamic> bundle = JSON.decode(file.readAsStringSync()); for (String key in bundle.keys) { // The ARB file resource "attributes" for foo are called @foo. if (key.startsWith('@')) - continue; - resources[key] = bundle[key]; + attributes[key.substring(1)] = bundle[key]; + else + resources[key] = bundle[key]; } } @@ -121,6 +132,7 @@ processBundle(new File(path), locale); } } + validateLocalizations(localeToResources, localeToResourceAttributes); final String regenerate = 'dart dev/tools/gen_localizations.dart ${directory.path} ${args[1]}'; print(outputHeader.replaceFirst('@(regenerate)', regenerate));
diff --git a/dev/tools/localizations_validator.dart b/dev/tools/localizations_validator.dart new file mode 100644 index 0000000..90aec93 --- /dev/null +++ b/dev/tools/localizations_validator.dart
@@ -0,0 +1,91 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +/// Enforces the following invariants in our localizations: +/// +/// - Resource keys are valid, i.e. they appear in the canonical list. +/// - Resource keys are complete for language-level locales, e.g. "es", "he". +/// +/// Uses "en" localizations as the canonical source of locale keys that other +/// locales are compared against. +/// +/// If validation fails, print an error message to STDERR and quit with exit +/// code 1. +void validateLocalizations( + Map<String, Map<String, String>> localeToResources, + Map<String, Map<String, dynamic>> localeToAttributes, +) { + final Map<String, String> canonicalLocalizations = localeToResources['en']; + final Set<String> canonicalKeys = new Set<String>.from(canonicalLocalizations.keys); + final StringBuffer errorMessages = new StringBuffer(); + bool explainMissingKeys = false; + for (final String locale in localeToResources.keys) { + final Map<String, String> resources = localeToResources[locale]; + + // Whether `key` corresponds to one of the plural variations of a key with + // the same prefix and suffix "Other". + // + // Many languages require only a subset of these variations, so we do not + // require them so long as the "Other" variation exists. + bool isPluralVariation(String key) { + final RegExp pluralRegexp = new RegExp(r'(\w*)(Zero|One|Two|Few|Many)$'); + final Match pluralMatch = pluralRegexp.firstMatch(key); + + if (pluralMatch == null) + return false; + + final String prefix = pluralMatch[1]; + return resources.containsKey('${prefix}Other'); + } + + final Set<String> keys = new Set<String>.from( + resources.keys.where((String key) => !isPluralVariation(key)) + ); + + // Make sure keys are valid (i.e. they also exist in the canonical + // localizations) + final Set<String> invalidKeys = keys.difference(canonicalKeys); + if (invalidKeys.isNotEmpty) + errorMessages.writeln('Locale "$locale" contains invalid resource keys: ${invalidKeys.join(', ')}'); + + // For language-level locales only, check that they have a complete list of + // keys, or opted out of using certain ones. + if (locale.length == 2) { + final Map<String, dynamic> attributes = localeToAttributes[locale]; + final List<String> missingKeys = <String>[]; + + for (final String missingKey in canonicalKeys.difference(keys)) { + final dynamic attribute = attributes[missingKey]; + final bool intentionallyOmitted = attribute is Map && attribute.containsKey('notUsed'); + if (!intentionallyOmitted && !isPluralVariation(missingKey)) + missingKeys.add(missingKey); + } + if (missingKeys.isNotEmpty) { + explainMissingKeys = true; + errorMessages.writeln('Locale "$locale" is missing the following resource keys: ${missingKeys.join(', ')}'); + } + } + } + + if (errorMessages.isNotEmpty) { + if (explainMissingKeys) { + errorMessages + ..writeln() + ..writeln( + 'If a resource key is intentionally omitted, add an attribute corresponding ' + 'to the key name with a "notUsed" property explaining why. Example:' + ) + ..writeln() + ..writeln('"@anteMeridiemAbbreviation": {') + ..writeln(' "notUsed": "Sindhi time format does not use a.m. indicator"') + ..writeln('}'); + } + + stderr.writeln('ERROR:'); + stderr.writeln(errorMessages); + exit(1); + } +}