blob: d0ed48be269a16c8c615d783423257860d9eae79 [file] [log] [blame]
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io' as io;
import 'package:args/args.dart';
import 'package:http/http.dart';
import 'package:path/path.dart' as path;
import 'browser_lock.dart';
import 'common.dart';
import 'utils.dart';
final ArgParser _argParser = ArgParser(allowTrailingOptions: false)
..addFlag(
'dry-run',
help: 'Whether or not to push changes to CIPD. When --dry-run is set, the '
'script will download everything and attempt to prepare the bundle '
'but will stop before publishing. When not set, the bundle will be '
'published.',
negatable: false,
)..addFlag(
'verbose',
abbr: 'v',
help: 'Enable verbose output.',
negatable: false,
);
late final bool dryRun;
late final bool verbose;
final Client _client = Client();
/// Rolls browser CIPD packages to the version specified in `browser_lock.yaml`.
///
/// Currently only rolls Chrome.
///
/// Chrome rolls are consumed by the "chrome_and_driver" and "chrome" LUCI recipes, here:
/// * https://cs.opensource.google/flutter/recipes/+/main:recipe_modules/flutter_deps/api.py;l=146
/// * https://cs.opensource.google/flutter/recipes/+/master:recipe_modules/web_util/api.py;l=22
///
/// Chromedriver is consumed by the same "chrome_and_driver" LUCI recipe, but also "chrome_driver":
/// * https://cs.opensource.google/flutter/recipes/+/master:recipe_modules/web_util/api.py;l=48
///
/// There's a small difference in the layout of the zip file coming from CIPD for the
/// Mac platform. In `Linux` and `Windows`, the chrome(.exe) executable is expected
/// to be placed directly in the root of the zip file.
///
/// However in `Mac`, the `Chromium.app` is expected to be placed inside of a
/// `chrome-mac` directory in the resulting zip file.
///
/// This script respects that historical quirk when building the CIPD packages.
/// In order for all the packages to be the same, the recipes listed above should
/// be made slightly smarter, so they can find the CHROME_EXECUTABLE in the right
/// place.
///
/// All platforms expect the "chromedriver" executable to be placed in the root
/// of the CIPD zip.
Future<void> main(List<String> args) async {
try {
processArgs(_argParser.parse(args));
await _BrowserRoller().roll();
io.exitCode = 0;
} on FormatException catch (e) {
print('''
Error! ${e.message}
Available options:
${_argParser.usage}
''');
io.exitCode = 1;
} finally {
_client.close();
}
}
// Initialize globals from the parsed command-line arguments.
void processArgs(ArgResults args) {
dryRun = args['dry-run'] as bool;
verbose = args['verbose'] as bool;
}
class _BrowserRoller {
_BrowserRoller();
final io.Directory _rollDir = io.Directory.systemTemp.createTempSync('browser-roll-');
final Map<String, PlatformBinding> _platformBindings = <String, PlatformBinding>{
'linux': LinuxPlatformBinding(),
'mac': MacPlatformBinding(),
'windows': WindowsPlatformBinding(),
};
final BrowserLock _lock = BrowserLock();
// Prints output when --verbose is set.
void vprint(String out) {
if (verbose) {
print(out);
}
}
// Roll Chromium and ChromeDriver for each of the Platforms.
Future<void> roll() async {
for (final MapEntry<String, PlatformBinding> entry in _platformBindings.entries) {
final String platform = entry.key;
final PlatformBinding binding = entry.value;
await _rollChromium(platform, binding);
await _rollChromeDriver(platform, binding);
}
if (dryRun) {
print('\nDry Run Done!\nNon-published roll artifacts kept here: ${_rollDir.path}\n');
} else {
// Clean-up
vprint('\nDeleting temporary directory: ${_rollDir.path}');
await _rollDir.delete(recursive: true);
print('\nDone.\n');
}
}
// Returns the contents for the CIPD config required to publish a new chromium package.
String _getCipdChromiumConfig({
required String package,
required String majorVersion,
required String buildId,
required String root,
}) {
return '''
package: $package
description: Chromium $majorVersion (build $buildId) used for testing
preserve_writable: true
root: $root
data:
- dir: .
''';
}
// Returns the contents for the CIPD config required to publish a new chromedriver package.
String _getCipdChromedriverConfig({
required String package,
required String majorVersion,
required String buildId,
required String root,
}) {
return '''
package: $package
description: Chromedriver for Chromium $majorVersion (build $buildId) used for testing
preserve_writable: true
root: $root
data:
- dir: .
''';
}
// Download a file from the internet, and put it in a temporary location.
Future<io.File> _downloadTemporaryFile(String url) async {
// Use the hash of the Url to temporarily store a file under tmp
final io.File downloadedFile = io.File(path.join(
io.Directory.systemTemp.path,
'download_${url.hashCode.toRadixString(16)}',
));
vprint(' Downloading [$url] into [${downloadedFile.path}]');
final StreamedResponse download = await _client.send(
Request('GET', Uri.parse(url)),
);
await download.stream.pipe(downloadedFile.openWrite());
return downloadedFile;
}
// Unzips a `file` into a `destination` Directory (must exist).
Future<void> _unzipAndDeleteFile(io.File zipFile, io.Directory destination) async {
vprint(' Unzipping [${zipFile.path}] into [$destination]');
await runProcess('unzip', <String>[
if (!verbose) ...<String>[
'-q',
],
zipFile.path,
'-d',
destination.path,
]);
vprint(' Deleting [${zipFile.path}]');
await zipFile.delete();
}
// Write String `contents` to a file in `path`.
//
// This is used to write CIPD config files to disk.
Future<io.File> _writeFile(String path, String contents) async {
vprint(' Writing file [$path]');
final io.File file = io.File(path, );
await file.writeAsString(contents);
return file;
}
// Locate the first subdirectory that contains more than one file under `root`.
// (or one ".app" bundle for mac)
//
// When uncompressing files, unzip might create some extra directories, but it
// seems that our scripts want our CIPD packages to contain everything in the root.
Future<io.Directory?> _locateContentRoot(io.Directory root) async {
final List<io.FileSystemEntity> children = root.listSync(followLinks: false);
assert(children.isNotEmpty);
if (root.path.toLowerCase().endsWith('.app')) {
// We've gone inside the .app bundle of the mac version!
return root.parent;
}
if (children.length == 1) {
if (children.first is io.Directory) {
return _locateContentRoot(children.first as io.Directory);
} else {
return root;
}
}
return root;
}
// Runs a CIPD command to upload a package defined by its `config` file.
Future<int> _uploadToCipd({
required io.File config,
required String version,
required String buildId,
}) {
final String cipdCommand = dryRun ? 'pkg-build' : 'create';
// CIPD won't fully shut up even in 'error' mode
final String logLevel = verbose ? 'debug' : 'warning';
vprint(' Running CIPD $cipdCommand');
return runProcess('cipd', <String>[
cipdCommand,
'--pkg-def',
path.basename(config.path),
'--json-output',
'${path.basenameWithoutExtension(config.path)}.json',
'--log-level',
logLevel,
if (!dryRun) ...<String>[
'--tag',
'version:$version',
'--ref',
buildId,
],
if (dryRun) ...<String>[
'--out',
'${path.basenameWithoutExtension(config.path)}.zip',
],
], workingDirectory: _rollDir.path);
}
// Determine if a `package` tagged with version:`versionTag` already exists in CIPD.
Future<bool> _cipdKnowsPackageVersion({
required String package,
required String versionTag,
}) async {
// $ cipd search $package -tag version:$versionTag
// Instances:
// $package:CIPD_PACKAGE_ID
// or:
// No matching instances.
final String logLevel = verbose ? 'debug' : 'warning';
vprint(' Searching for $package version:$versionTag in CIPD');
final String stdout = await evalProcess('cipd', <String>[
'search',
package,
'--tag',
'version:$versionTag',
'--log-level',
logLevel,
], workingDirectory: _rollDir.path);
return stdout.contains('Instances:') && stdout.contains(package);
}
// Downloads Chromium from the internet, packs it in the directory structure
// that the LUCI script wants. The result of this will be then uploaded to CIPD.
Future<void> _rollChromium(String platform, PlatformBinding binding) async {
final String chromeBuild = binding.getChromeBuild(_lock.chromeLock);
final String majorVersion = _lock.chromeLock.version;
final String url = binding.getChromeDownloadUrl(chromeBuild);
final String cipdPackageName = 'flutter_internal/browsers/chrome/$platform-amd64';
final io.Directory platformDir = io.Directory(path.join(_rollDir.path, platform));
print('\nRolling Chromium for $platform (version:$majorVersion, build $chromeBuild)');
// Bail out if CIPD already has version:$majorVersion for this package!
if (!dryRun && await _cipdKnowsPackageVersion(package: cipdPackageName, versionTag: majorVersion)) {
print(' Skipping $cipdPackageName version:$majorVersion. Already uploaded to CIPD!');
vprint(' Update browser_lock.yaml and use a different version value.');
return;
}
await platformDir.create(recursive: true);
vprint(' Created target directory [${platformDir.path}]');
final io.File chromeDownload = await _downloadTemporaryFile(url);
await _unzipAndDeleteFile(chromeDownload, platformDir);
late String relativePlatformDirPath;
// Preserve the `chrome-mac` directory when bundling, but remove it for win and linux.
if (platform == 'Mac') {
relativePlatformDirPath = path.relative(platformDir.path, from: _rollDir.path);
} else {
final io.Directory? actualContentRoot = await _locateContentRoot(platformDir);
assert(actualContentRoot != null);
relativePlatformDirPath = path.relative(actualContentRoot!.path, from: _rollDir.path);
}
// Create the config manifest to upload to CIPD
final io.File cipdConfigFile = await _writeFile(
path.join(_rollDir.path, 'cipd.chromium.$platform.yaml'),
_getCipdChromiumConfig(
package: cipdPackageName,
majorVersion: majorVersion,
buildId: chromeBuild,
root: relativePlatformDirPath,
));
// Run CIPD
await _uploadToCipd(config: cipdConfigFile, version: majorVersion, buildId: chromeBuild);
}
// Downloads Chromedriver from the internet, packs it in the directory structure
// that the LUCI script wants. The result of this will be then uploaded to CIPD.
Future<void> _rollChromeDriver(String platform, PlatformBinding binding) async {
final String chromeBuild = binding.getChromeBuild(_lock.chromeLock);
final String majorVersion = _lock.chromeLock.version;
final String url = binding.getChromeDriverDownloadUrl(chromeBuild);
final String cipdPackageName = 'flutter_internal/browser-drivers/chrome/$platform-amd64';
final io.Directory platformDir = io.Directory(path.join(_rollDir.path, '${platform}_driver'));
print('\nRolling Chromedriver for $platform (version:$majorVersion, build $chromeBuild)');
// Bail out if CIPD already has version:$majorVersion for this package!
if (!dryRun && await _cipdKnowsPackageVersion(package: cipdPackageName, versionTag: majorVersion)) {
print(' Skipping $cipdPackageName version:$majorVersion. Already uploaded to CIPD!');
vprint(' Update browser_lock.yaml and use a different version value.');
return;
}
await platformDir.create(recursive: true);
vprint(' Created target directory [${platformDir.path}]');
final io.File chromedriverDownload = await _downloadTemporaryFile(url);
await _unzipAndDeleteFile(chromedriverDownload, platformDir);
// Ensure the chromedriver executable is placed in the root of the bundle.
final io.Directory? actualContentRoot = await _locateContentRoot(platformDir);
assert(actualContentRoot != null);
final String relativePlatformDirPath = path.relative(actualContentRoot!.path, from: _rollDir.path);
// Create the config manifest to upload to CIPD
final io.File cipdConfigFile = await _writeFile(
path.join(_rollDir.path, 'cipd.chromedriver.$platform.yaml'),
_getCipdChromedriverConfig(
package: cipdPackageName,
majorVersion: majorVersion,
buildId: chromeBuild,
root: relativePlatformDirPath,
));
// Run CIPD
await _uploadToCipd(config: cipdConfigFile, version: majorVersion, buildId: chromeBuild);
}
}