Added a check to the analyzer script to detect skipped tests. (#88003) Added a check to the analyzer script to detect skipped tests that aren't commented. The comment following the `skip` parameter should include either a link to a github issue tracking the reenabling of the test, or a '[intended]' tag with a brief description of why the test should never be enabled for the given condition.
diff --git a/dev/bots/analyze.dart b/dev/bots/analyze.dart index 0fa978b..7323a5b 100644 --- a/dev/bots/analyze.dart +++ b/dev/bots/analyze.dart
@@ -60,6 +60,9 @@ print('$clock Deprecations...'); await verifyDeprecations(flutterRoot); + print('$clock Skip test comments...'); + await verifySkipTestComments(flutterRoot); + print('$clock Licenses...'); await verifyNoMissingLicense(flutterRoot); @@ -280,6 +283,38 @@ } } +final RegExp _skipTestCommentPattern = RegExp(r'\bskip:.*?//(.*)'); +const Pattern _skipTestIntentionalPattern = '[intended]'; +final Pattern _skipTestTrackingBugPattern = RegExp(r'https+?://github.com/.*/issues/[0-9]+'); + +Future<void> verifySkipTestComments(String workingDirectory) async { + final List<String> errors = <String>[]; + final Stream<File> testFiles = _allFiles(workingDirectory, 'dart', minimumMatches: 1500) + .where((File f) => f.path.endsWith('_test.dart')); + + await for (final File file in testFiles) { + final List<String> lines = file.readAsLinesSync(); + for (int index = 0; index < lines.length; index++) { + final Match? match = _skipTestCommentPattern.firstMatch(lines[index]); + final String? skipComment = match?.group(1); + if (skipComment != null + && !skipComment.contains(_skipTestIntentionalPattern) + && !skipComment.contains(_skipTestTrackingBugPattern)) { + final int sourceLine = index + 1; + errors.add('${file.path}:$sourceLine}: skip test without a justification comment.'); + } + } + } + + // Fail if any errors + if (errors.isNotEmpty) { + exitWithError(<String>[ + ...errors, + '\n${bold}See: https://github.com/flutter/flutter/wiki/Tree-hygiene#skipped-tests$reset', + ]); + } +} + final RegExp _testImportPattern = RegExp(r'''import (['"])([^'"]+_test\.dart)\1'''); const Set<String> _exemptTestImports = <String>{ 'package:flutter_test/flutter_test.dart',
diff --git a/dev/devicelab/bin/tasks/technical_debt__cost.dart b/dev/devicelab/bin/tasks/technical_debt__cost.dart index 146d764..34fe474 100644 --- a/dev/devicelab/bin/tasks/technical_debt__cost.dart +++ b/dev/devicelab/bin/tasks/technical_debt__cost.dart
@@ -58,7 +58,7 @@ total += deprecationCost; if (line.contains(legacyDeprecationPattern)) total += legacyDeprecationCost; - if (isTest && line.contains('skip:')) + if (isTest && line.contains('skip:') && !line.contains('[intended]')) total += skipCost; if (isDart && isOptingOutOfNullSafety(line)) total += fileNullSafetyMigrationCost;