Fix lexer issue where select/plural/other/underscores cannot be in identifier names. (#119190)
diff --git a/packages/flutter_tools/lib/src/localizations/message_parser.dart b/packages/flutter_tools/lib/src/localizations/message_parser.dart
index 19a00d7..c2bd7d9 100644
--- a/packages/flutter_tools/lib/src/localizations/message_parser.dart
+++ b/packages/flutter_tools/lib/src/localizations/message_parser.dart
@@ -158,20 +158,14 @@
RegExp brace = RegExp(r'{|}');
RegExp whitespace = RegExp(r'\s+');
-RegExp pluralKeyword = RegExp(r'plural');
-RegExp selectKeyword = RegExp(r'select');
-RegExp otherKeyword = RegExp(r'other');
RegExp numeric = RegExp(r'[0-9]+');
-RegExp alphanumeric = RegExp(r'[a-zA-Z0-9]+');
+RegExp alphanumeric = RegExp(r'[a-zA-Z0-9|_]+');
RegExp comma = RegExp(r',');
RegExp equalSign = RegExp(r'=');
// List of token matchers ordered by precedence
Map<ST, RegExp> matchers = <ST, RegExp>{
ST.empty: whitespace,
- ST.plural: pluralKeyword,
- ST.select: selectKeyword,
- ST.other: otherKeyword,
ST.number: numeric,
ST.comma: comma,
ST.equalSign: equalSign,
@@ -303,12 +297,25 @@
// Do not add whitespace as a token.
startIndex = match.end;
continue;
- } else if (<ST>[ST.plural, ST.select].contains(matchedType) && tokens.last.type == ST.openBrace) {
- // Treat "plural" or "select" as identifier if it comes right after an open brace.
+ } else if (<ST>[ST.identifier].contains(matchedType) && tokens.last.type == ST.openBrace) {
+ // Treat any token as identifier if it comes right after an open brace, whether it's a keyword or not.
tokens.add(Node(ST.identifier, startIndex, value: match.group(0)));
startIndex = match.end;
continue;
} else {
+ // Handle keywords separately. Otherwise, lexer will assume parts of identifiers may be keywords.
+ final String tokenStr = match.group(0)!;
+ switch(tokenStr) {
+ case 'plural':
+ matchedType = ST.plural;
+ break;
+ case 'select':
+ matchedType = ST.select;
+ break;
+ case 'other':
+ matchedType = ST.other;
+ break;
+ }
tokens.add(Node(matchedType!, startIndex, value: match.group(0)));
startIndex = match.end;
continue;
diff --git a/packages/flutter_tools/test/general.shard/message_parser_test.dart b/packages/flutter_tools/test/general.shard/message_parser_test.dart
index 5c36d59..4d19d18 100644
--- a/packages/flutter_tools/test/general.shard/message_parser_test.dart
+++ b/packages/flutter_tools/test/general.shard/message_parser_test.dart
@@ -226,6 +226,22 @@
expect(tokens[5].type, equals(ST.identifier));
});
+ testWithoutContext('lexer identifier names can contain underscores', () {
+ final List<Node> tokens = Parser('keywords', 'app_en.arb', '{ test_placeholder } { test_select, select, singular{test} other{hmm} }').lexIntoTokens();
+ expect(tokens[1].value, equals('test_placeholder'));
+ expect(tokens[1].type, equals(ST.identifier));
+ expect(tokens[5].value, equals('test_select'));
+ expect(tokens[5].type, equals(ST.identifier));
+ });
+
+ testWithoutContext('lexer identifier names can contain the strings select or plural', () {
+ final List<Node> tokens = Parser('keywords', 'app_en.arb', '{ selectTest } { pluralTest, select, singular{test} other{hmm} }').lexIntoTokens();
+ expect(tokens[1].value, equals('selectTest'));
+ expect(tokens[1].type, equals(ST.identifier));
+ expect(tokens[5].value, equals('pluralTest'));
+ expect(tokens[5].type, equals(ST.identifier));
+ });
+
testWithoutContext('lexer: lexically correct but syntactically incorrect', () {
final List<Node> tokens = Parser(
'syntax',
