blob: 5cf5f25db92f2bdd9d770d9c91e2a41892598884 [file] [log] [blame]
// Copyright 2020 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:convert';
import 'dart:core';
import 'dart:io';
import 'package:file/file.dart';
import 'package:file/local.dart';
import 'package:path/path.dart' as path;
const FileSystem fs = LocalFileSystem();
// Cocoon's root is the parent of the current working directory,
final Directory cocoonRoot = fs.currentDirectory.parent;
Future<void> main(List<String> arguments) async {
print('STARTING ANALYSIS');
print('cocoonRoot: ${cocoonRoot.path}');
await run(arguments);
print('Analysis successful.');
}
Future<void> run(List<String> arguments) async {
bool assertsEnabled = false;
assert(() {
assertsEnabled = true;
return true;
}());
if (!assertsEnabled) {
exitWithError(<String>['The analyze.dart script must be run with --enable-asserts.']);
}
print('Trailing spaces...');
await verifyNoTrailingSpaces(cocoonRoot.path);
print('Executable allowlist...');
await _checkForNewExecutables();
print('Proto analysis...');
await verifyProtos(cocoonRoot);
}
// TESTS
Future<void> verifyNoTrailingSpaces(
String workingDirectory, {
int minimumMatches = 100,
}) async {
final List<File> files = await _allFiles(workingDirectory, null, minimumMatches: minimumMatches)
.where((File file) => path.basename(file.path) != 'serviceaccount.enc')
.where((File file) => path.basename(file.path) != 'Ahem.ttf')
.where((File file) => path.extension(file.path) != '.snapshot')
.where((File file) => path.extension(file.path) != '.png')
.where((File file) => path.extension(file.path) != '.jpg')
.where((File file) => path.extension(file.path) != '.ico')
.where((File file) => path.extension(file.path) != '.jar')
.where((File file) => path.extension(file.path) != '.swp')
.where((File file) => !path.basename(file.path).endsWith('pbserver.dart'))
.where((File file) => !path.basename(file.path).endsWith('pb.dart'))
.where((File file) => !path.basename(file.path).endsWith('pbenum.dart'))
.toList();
final List<String> problems = <String>[];
for (final File file in files) {
final List<String> lines = file.readAsLinesSync();
for (int index = 0; index < lines.length; index += 1) {
if (lines[index].endsWith(' ')) {
problems.add('${file.path}:${index + 1}: trailing U+0020 space character');
} else if (lines[index].endsWith('\t')) {
problems.add('${file.path}:${index + 1}: trailing U+0009 tab character');
}
}
if (lines.isNotEmpty && lines.last == '') problems.add('${file.path}:${lines.length}: trailing blank line');
}
if (problems.isNotEmpty) exitWithError(problems);
}
Future<void> verifyProtos(Directory workingDirectory) async {
final List<String> errors = <String>[];
final List<File> protos = await _allFiles(workingDirectory.path, 'proto', minimumMatches: 1).toList();
for (final File proto in protos) {
final String content = proto.readAsStringSync();
if (!content.contains(RegExp(r'package\ \w+;'))) {
errors.add('${proto.path} requires a package (https://protobuf.dev/programming-guides/proto2/#packages)');
}
}
if (errors.isNotEmpty) {
exitWithError(<String>[
'The following files are missing package declarations:',
...errors,
]);
}
}
// UTILITY FUNCTIONS
Future<List<File>> _gitFiles(String workingDirectory, {bool runSilently = true}) async {
final EvalResult evalResult = await _evalCommand(
'git',
<String>['ls-files', '-z'],
workingDirectory: workingDirectory,
runSilently: runSilently,
);
if (evalResult.exitCode != 0) {
exitWithError(<String>[
'git ls-files failed with exit code ${evalResult.exitCode}',
'stdout:',
evalResult.stdout,
'stderr:',
evalResult.stderr,
]);
}
final List<String> filenames = evalResult.stdout.split('\x00');
assert(filenames.last.isEmpty); // git ls-files gives a trailing blank 0x00
filenames.removeLast();
return filenames.map<File>((String filename) => fs.file(path.join(workingDirectory, filename))).toList();
}
Stream<File> _allFiles(String workingDirectory, String? extension, {required int minimumMatches}) async* {
final Set<String> gitFileNamesSet = <String>{};
gitFileNamesSet.addAll((await _gitFiles(workingDirectory)).map((File f) => path.canonicalize(f.absolute.path)));
assert(extension == null || !extension.startsWith('.'), 'Extension argument should not start with a period.');
final Set<FileSystemEntity> pending = <FileSystemEntity>{fs.directory(workingDirectory)};
int matches = 0;
while (pending.isNotEmpty) {
final FileSystemEntity entity = pending.first;
pending.remove(entity);
if (path.extension(entity.path) == '.tmpl') continue;
if (entity is File) {
if (!gitFileNamesSet.contains(path.canonicalize(entity.absolute.path))) continue;
if (path.basename(entity.path) == 'flutter_export_environment.sh') continue;
if (path.basename(entity.path) == 'gradlew.bat') continue;
if (path.basename(entity.path) == '.DS_Store') continue;
if (extension == null || path.extension(entity.path) == '.$extension') {
matches += 1;
yield entity;
}
} else if (entity is Directory) {
if (fs.file(path.join(entity.path, '.dartignore')).existsSync()) continue;
if (path.basename(entity.path) == '.git') continue;
if (path.basename(entity.path) == '.idea') continue;
if (path.basename(entity.path) == '.gradle') continue;
if (path.basename(entity.path) == '.dart_tool') continue;
if (path.basename(entity.path) == '.idea') continue;
if (path.basename(entity.path) == 'build') continue;
pending.addAll(entity.listSync());
}
}
assert(
matches >= minimumMatches,
'Expected to find at least $minimumMatches files with extension ".$extension" in "$workingDirectory", but only found $matches.',
);
}
class EvalResult {
EvalResult({
required this.stdout,
required this.stderr,
this.exitCode = 0,
});
final String stdout;
final String stderr;
final int exitCode;
}
Future<EvalResult> _evalCommand(
String executable,
List<String> arguments, {
required String workingDirectory,
Map<String, String>? environment,
bool allowNonZeroExit = false,
bool runSilently = false,
}) async {
final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}';
final String relativeWorkingDir = path.relative(workingDirectory);
if (!runSilently) {
print('RUNNING $relativeWorkingDir $commandDescription');
}
final Stopwatch time = Stopwatch()..start();
final Process process = await Process.start(
executable,
arguments,
workingDirectory: workingDirectory,
environment: environment,
);
final Future<List<List<int>>> savedStdout = process.stdout.toList();
final Future<List<List<int>>> savedStderr = process.stderr.toList();
final int exitCode = await process.exitCode;
final EvalResult result = EvalResult(
stdout: utf8.decode((await savedStdout).expand<int>((List<int> ints) => ints).toList()),
stderr: utf8.decode((await savedStderr).expand<int>((List<int> ints) => ints).toList()),
exitCode: exitCode,
);
if (!runSilently) {
print('ELAPSED TIME: ${time.elapsed} for $commandDescription in $relativeWorkingDir');
}
if (exitCode != 0 && !allowNonZeroExit) {
stderr.write(result.stderr);
exitWithError(<String>[
'ERROR: Last command exited with $exitCode.',
'Command: $commandDescription',
'Relative working directory: $relativeWorkingDir',
]);
}
return result;
}
// These files legitimately require executable permissions
const Set<String> kExecutableAllowlist = <String>{
'app_dart/tool/build.sh',
'cipd_packages/codesign/tool/build.sh',
'cipd_packages/device_doctor/tool/build.sh',
'cipd_packages/doxygen/tool/build.sh',
'cloud_build/dashboard_build.sh',
'cloud_build/deploy_app_dart.sh',
'cloud_build/deploy_auto_submit.sh',
'cloud_build/deploy_cron_jobs.sh',
'cloud_build/get_docker_image_provenance.sh',
'cloud_build/verify_provenance.sh',
'dashboard/regen_mocks.sh',
'dev/provision_salt.sh',
'dev/prs_to_main.sh',
'format.sh',
'packages/buildbucket-dart/tool/regenerate.sh',
'test_utilities/bin/analyze.sh',
'test_utilities/bin/config_test_runner.sh',
'test_utilities/bin/dart_test_runner.sh',
'test_utilities/bin/flutter_test_runner.sh',
'test_utilities/bin/global_test_runner.dart',
'test_utilities/bin/licenses.sh',
'test_utilities/bin/prepare_environment.sh',
};
const String kShebangRegex = r'#!/usr/bin/env (bash|sh)';
Future<void> _checkForNewExecutables() async {
// 0b001001001
const int executableBitMask = 0x49;
final List<File> files = await _gitFiles(cocoonRoot.path);
final List<String> relativePaths = files.map<String>((File file) {
return path.relative(
file.path,
from: cocoonRoot.path,
);
}).toList();
for (String allowed in kExecutableAllowlist) {
if (!relativePaths.contains(allowed)) {
throw Exception(
'File $allowed in kExecutableAllowlist in analyze/analyze.dart '
'does not exist. Please fix path or remove from kExecutableAllowlist.',
);
}
}
int unexpectedExecutableCount = 0;
int unexpectedShebangShellCount = 0;
for (final File file in files) {
final String relativePath = path.relative(
file.path,
from: cocoonRoot.path,
);
final FileStat stat = file.statSync();
final bool isExecutable = stat.mode & executableBitMask != 0x0;
final bool inAllowList = kExecutableAllowlist.contains(relativePath);
if (isExecutable && !inAllowList) {
unexpectedExecutableCount += 1;
print('$relativePath is executable: ${(stat.mode & 0x1FF).toRadixString(2)}');
}
if (inAllowList && file.path.endsWith('.sh')) {
final String shebang = file.readAsLinesSync().first;
if (!shebang.startsWith(RegExp(kShebangRegex))) {
unexpectedShebangShellCount += 1;
print("$relativePath has the initial line of $shebang, which doesn't match '$kShebangRegex'");
}
}
}
if (unexpectedExecutableCount > 0) {
throw Exception(
'found $unexpectedExecutableCount unexpected executable file'
'${unexpectedExecutableCount == 1 ? '' : 's'}! If this was intended, you '
'must add this file to kExecutableAllowlist in analyze/analyze.dart',
);
}
if (unexpectedShebangShellCount > 0) {
throw Exception(
'found $unexpectedShebangShellCount unexpected shell #! line'
'${unexpectedShebangShellCount == 1 ? '' : 's'}! If this was intended, you '
'must modify kShebangRegex in analyze/analyze.dart',
);
}
}
void exitWithError(List<String> messages) {
final String line = '━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━';
print(line);
messages.forEach(print);
print(line);
exit(1);
}