blob: 7870c7807f926bb81140763f9fc58a21aae14965 [file] [log] [blame]
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:crypto/crypto.dart';
import 'package:file/memory.dart';
import 'package:meta/meta.dart';
import 'package:process/process.dart';
import 'base/common.dart';
import 'base/error_handling_io.dart';
import 'base/file_system.dart';
import 'base/io.dart' show HttpClient, HttpClientRequest, HttpClientResponse, HttpHeaders, HttpStatus, SocketException;
import 'base/logger.dart';
import 'base/net.dart';
import 'base/os.dart' show OperatingSystemUtils;
import 'base/platform.dart';
import 'base/terminal.dart';
import 'base/user_messages.dart';
import 'build_info.dart';
import 'convert.dart';
import 'features.dart';
const String kFlutterRootEnvironmentVariableName = 'FLUTTER_ROOT'; // should point to //flutter/ (root of flutter/flutter repo)
const String kFlutterEngineEnvironmentVariableName = 'FLUTTER_ENGINE'; // should point to //engine/src/ (root of flutter/engine repo)
const String kSnapshotFileName = 'flutter_tools.snapshot'; // in //flutter/bin/cache/
const String kFlutterToolsScriptFileName = 'flutter_tools.dart'; // in //flutter/packages/flutter_tools/bin/
const String kFlutterEnginePackageName = 'sky_engine';
/// A tag for a set of development artifacts that need to be cached.
class DevelopmentArtifact {
const DevelopmentArtifact._(this.name, {this.feature});
/// The name of the artifact.
///
/// This should match the flag name in precache.dart.
final String name;
/// A feature to control the visibility of this artifact.
final Feature? feature;
/// Artifacts required for Android development.
static const DevelopmentArtifact androidGenSnapshot = DevelopmentArtifact._('android_gen_snapshot', feature: flutterAndroidFeature);
static const DevelopmentArtifact androidMaven = DevelopmentArtifact._('android_maven', feature: flutterAndroidFeature);
// Artifacts used for internal builds.
static const DevelopmentArtifact androidInternalBuild = DevelopmentArtifact._('android_internal_build', feature: flutterAndroidFeature);
/// Artifacts required for iOS development.
static const DevelopmentArtifact iOS = DevelopmentArtifact._('ios', feature: flutterIOSFeature);
/// Artifacts required for web development.
static const DevelopmentArtifact web = DevelopmentArtifact._('web', feature: flutterWebFeature);
/// Artifacts required for desktop macOS.
static const DevelopmentArtifact macOS = DevelopmentArtifact._('macos', feature: flutterMacOSDesktopFeature);
/// Artifacts required for desktop Windows.
static const DevelopmentArtifact windows = DevelopmentArtifact._('windows', feature: flutterWindowsDesktopFeature);
/// Artifacts required for desktop Linux.
static const DevelopmentArtifact linux = DevelopmentArtifact._('linux', feature: flutterLinuxDesktopFeature);
/// Artifacts required for Fuchsia.
static const DevelopmentArtifact fuchsia = DevelopmentArtifact._('fuchsia', feature: flutterFuchsiaFeature);
/// Artifacts required for the Flutter Runner.
static const DevelopmentArtifact flutterRunner = DevelopmentArtifact._('flutter_runner', feature: flutterFuchsiaFeature);
/// Artifacts required for desktop Windows UWP.
static const DevelopmentArtifact windowsUwp = DevelopmentArtifact._('winuwp', feature: windowsUwpEmbedding);
/// Artifacts required for any development platform.
///
/// This does not need to be explicitly returned from requiredArtifacts as
/// it will always be downloaded.
static const DevelopmentArtifact universal = DevelopmentArtifact._('universal');
/// The values of DevelopmentArtifacts.
static final List<DevelopmentArtifact> values = <DevelopmentArtifact>[
androidGenSnapshot,
androidMaven,
androidInternalBuild,
iOS,
web,
macOS,
windows,
linux,
fuchsia,
universal,
flutterRunner,
windowsUwp,
];
@override
String toString() => 'Artifact($name)';
}
/// A wrapper around the `bin/cache/` directory.
///
/// This does not provide any artifacts by default. See [FlutterCache] for the default
/// artifact set.
class Cache {
/// [rootOverride] is configurable for testing.
/// [artifacts] is configurable for testing.
Cache({
@protected Directory? rootOverride,
@protected List<ArtifactSet>? artifacts,
required Logger logger,
required FileSystem fileSystem,
required Platform platform,
required OperatingSystemUtils osUtils,
}) : _rootOverride = rootOverride,
_logger = logger,
_fileSystem = fileSystem,
_platform = platform,
_osUtils = osUtils,
_net = Net(logger: logger, platform: platform),
_fsUtils = FileSystemUtils(fileSystem: fileSystem, platform: platform),
_artifacts = artifacts ?? <ArtifactSet>[];
/// Create a [Cache] for testing.
///
/// Defaults to a memory file system, fake platform,
/// buffer logger, and no accessible artifacts.
/// By default, the root cache directory path is "cache".
factory Cache.test({
Directory? rootOverride,
List<ArtifactSet>? artifacts,
Logger? logger,
FileSystem? fileSystem,
Platform? platform,
required ProcessManager processManager,
}) {
fileSystem ??= rootOverride?.fileSystem ?? MemoryFileSystem.test();
platform ??= FakePlatform(environment: <String, String>{});
logger ??= BufferLogger.test();
return Cache(
rootOverride: rootOverride ??= fileSystem.directory('cache'),
artifacts: artifacts ?? <ArtifactSet>[],
logger: logger,
fileSystem: fileSystem,
platform: platform,
osUtils: OperatingSystemUtils(
fileSystem: fileSystem,
logger: logger,
platform: platform,
processManager: processManager,
),
);
}
final Logger _logger;
final Platform _platform;
final FileSystem _fileSystem;
final OperatingSystemUtils _osUtils;
final Directory? _rootOverride;
final List<ArtifactSet> _artifacts;
final Net _net;
final FileSystemUtils _fsUtils;
late final ArtifactUpdater _artifactUpdater = _createUpdater();
@protected
void registerArtifact(ArtifactSet artifactSet) {
_artifacts.add(artifactSet);
}
/// This has to be lazy because it requires FLUTTER_ROOT to be initialized.
ArtifactUpdater _createUpdater() {
return ArtifactUpdater(
operatingSystemUtils: _osUtils,
logger: _logger,
fileSystem: _fileSystem,
tempStorage: getDownloadDir(),
platform: _platform,
httpClient: HttpClient(),
);
}
static const List<String> _hostsBlockedInChina = <String> [
'storage.googleapis.com',
];
// Initialized by FlutterCommandRunner on startup.
// Explore making this field lazy to catch non-initialized access.
static String? flutterRoot;
/// Determine the absolute and normalized path for the root of the current
/// Flutter checkout.
///
/// This method has a series of fallbacks for determining the repo location. The
/// first success will immediately return the root without further checks.
///
/// The order of these tests is:
/// 1. FLUTTER_ROOT environment variable contains the path.
/// 2. Platform script is a data URI scheme, returning `../..` to support
/// tests run from `packages/flutter_tools`.
/// 3. Platform script is package URI scheme, returning the grandparent directory
/// of the package config file location from `packages/flutter_tools/.packages`.
/// 4. Platform script file path is the snapshot path generated by `bin/flutter`,
/// returning the grandparent directory from `bin/cache`.
/// 5. Platform script file name is the entrypoint in `packages/flutter_tools/bin/flutter_tools.dart`,
/// returning the 4th parent directory.
/// 6. The current directory
///
/// If an exception is thrown during any of these checks, an error message is
/// printed and `.` is returned by default (6).
static String defaultFlutterRoot({
required Platform platform,
required FileSystem fileSystem,
required UserMessages userMessages,
}) {
String normalize(String path) {
return fileSystem.path.normalize(fileSystem.path.absolute(path));
}
if (platform.environment.containsKey(kFlutterRootEnvironmentVariableName)) {
return normalize(platform.environment[kFlutterRootEnvironmentVariableName]!);
}
try {
if (platform.script.scheme == 'data') {
return normalize('../..'); // The tool is running as a test.
}
final String Function(String) dirname = fileSystem.path.dirname;
if (platform.script.scheme == 'package') {
final String packageConfigPath = Uri.parse(platform.packageConfig!).toFilePath(
windows: platform.isWindows,
);
return normalize(dirname(dirname(dirname(packageConfigPath))));
}
if (platform.script.scheme == 'file') {
final String script = platform.script.toFilePath(
windows: platform.isWindows,
);
if (fileSystem.path.basename(script) == kSnapshotFileName) {
return normalize(dirname(dirname(fileSystem.path.dirname(script))));
}
if (fileSystem.path.basename(script) == kFlutterToolsScriptFileName) {
return normalize(dirname(dirname(dirname(dirname(script)))));
}
}
} on Exception catch (error) {
// There is currently no logger attached since this is computed at startup.
// ignore: avoid_print
print(userMessages.runnerNoRoot('$error'));
}
return normalize('.');
}
// Whether to cache artifacts for all platforms. Defaults to only caching
// artifacts for the current platform.
bool includeAllPlatforms = false;
// Names of artifacts which should be cached even if they would normally
// be filtered out for the current platform.
Set<String>? platformOverrideArtifacts;
// Whether to cache the unsigned mac binaries. Defaults to caching the signed binaries.
bool useUnsignedMacBinaries = false;
static RandomAccessFile? _lock;
static bool _lockEnabled = true;
/// Turn off the [lock]/[releaseLock] mechanism.
///
/// This is used by the tests since they run simultaneously and all in one
/// process and so it would be a mess if they had to use the lock.
@visibleForTesting
static void disableLocking() {
_lockEnabled = false;
}
/// Turn on the [lock]/[releaseLock] mechanism.
///
/// This is used by the tests.
@visibleForTesting
static void enableLocking() {
_lockEnabled = true;
}
/// Check if lock acquired, skipping FLUTTER_ALREADY_LOCKED reentrant checks.
///
/// This is used by the tests.
@visibleForTesting
static bool isLocked() {
return _lock != null;
}
/// Lock the cache directory.
///
/// This happens while required artifacts are updated
/// (see [FlutterCommandRunner.runCommand]).
///
/// This uses normal POSIX flock semantics.
Future<void> lock() async {
if (!_lockEnabled) {
return;
}
assert(_lock == null);
final File lockFile =
_fileSystem.file(_fileSystem.path.join(flutterRoot!, 'bin', 'cache', 'lockfile'));
try {
_lock = lockFile.openSync(mode: FileMode.write);
} on FileSystemException catch (e) {
_logger.printError('Failed to open or create the artifact cache lockfile: "$e"');
_logger.printError('Please ensure you have permissions to create or open ${lockFile.path}');
throwToolExit('Failed to open or create the lockfile');
}
bool locked = false;
bool printed = false;
while (!locked) {
try {
_lock!.lockSync();
locked = true;
} on FileSystemException {
if (!printed) {
_logger.printTrace('Waiting to be able to obtain lock of Flutter binary artifacts directory: ${_lock!.path}');
// This needs to go to stderr to avoid cluttering up stdout if a parent
// process is collecting stdout. It's not really an "error" though,
// so print it in grey.
_logger.printError(
'Waiting for another flutter command to release the startup lock...',
color: TerminalColor.grey,
);
printed = true;
}
await Future<void>.delayed(const Duration(milliseconds: 50));
}
}
}
/// Releases the lock.
///
/// This happens automatically on startup (see [FlutterCommand.verifyThenRunCommand])
/// after the command's required artifacts are updated.
void releaseLock() {
if (!_lockEnabled || _lock == null) {
return;
}
_lock!.closeSync();
_lock = null;
}
/// Checks if the current process owns the lock for the cache directory at
/// this very moment; throws a [StateError] if it doesn't.
void checkLockAcquired() {
if (_lockEnabled && _lock == null && _platform.environment['FLUTTER_ALREADY_LOCKED'] != 'true') {
throw StateError(
'The current process does not own the lock for the cache directory. This is a bug in Flutter CLI tools.',
);
}
}
/// The current version of Dart used to build Flutter and run the tool.
String get dartSdkVersion {
if (_dartSdkVersion == null) {
// Make the version string more customer-friendly.
// Changes '2.1.0-dev.8.0.flutter-4312ae32' to '2.1.0 (build 2.1.0-dev.8.0 4312ae32)'
final String justVersion = _platform.version.split(' ')[0];
_dartSdkVersion = justVersion.replaceFirstMapped(RegExp(r'(\d+\.\d+\.\d+)(.+)'), (Match match) {
final String noFlutter = match[2]!.replaceAll('.flutter-', ' ');
return '${match[1]} (build ${match[1]}$noFlutter)';
});
}
return _dartSdkVersion!;
}
String? _dartSdkVersion;
/// The current version of Dart used to build Flutter and run the tool.
String get dartSdkBuild {
if (_dartSdkBuild == null) {
// Make the version string more customer-friendly.
// Changes '2.1.0-dev.8.0.flutter-4312ae32' to '2.1.0 (build 2.1.0-dev.8.0 4312ae32)'
final String justVersion = _platform.version.split(' ')[0];
_dartSdkBuild = justVersion.replaceFirstMapped(RegExp(r'(\d+\.\d+\.\d+)(.+)'), (Match match) {
final String noFlutter = match[2]!.replaceAll('.flutter-', ' ');
return '${match[1]}$noFlutter';
});
}
return _dartSdkBuild!;
}
String? _dartSdkBuild;
/// The current version of the Flutter engine the flutter tool will download.
String get engineRevision {
_engineRevision ??= getVersionFor('engine');
if (_engineRevision == null) {
throwToolExit('Could not determine engine revision.');
}
return _engineRevision!;
}
String? _engineRevision;
String get storageBaseUrl {
final String? overrideUrl = _platform.environment['FLUTTER_STORAGE_BASE_URL'];
if (overrideUrl == null) {
return 'https://storage.googleapis.com';
}
// verify that this is a valid URI.
try {
Uri.parse(overrideUrl);
} on FormatException catch (err) {
throwToolExit('"FLUTTER_STORAGE_BASE_URL" contains an invalid URI:\n$err');
}
_maybeWarnAboutStorageOverride(overrideUrl);
return overrideUrl;
}
String get cipdBaseUrl {
final String? overrideUrl = _platform.environment['FLUTTER_STORAGE_BASE_URL'];
if (overrideUrl == null) {
return 'https://chrome-infra-packages.appspot.com/dl';
}
final Uri original;
try {
original = Uri.parse(overrideUrl);
} on FormatException catch (err) {
throwToolExit('"FLUTTER_STORAGE_BASE_URL" contains an invalid URI:\n$err');
}
final String cipdOverride = original.replace(
pathSegments: <String>[
...original.pathSegments,
'flutter_infra_release',
'cipd',
],
).toString();
return cipdOverride;
}
bool _hasWarnedAboutStorageOverride = false;
void _maybeWarnAboutStorageOverride(String overrideUrl) {
if (_hasWarnedAboutStorageOverride) {
return;
}
_logger.printStatus(
'Flutter assets will be downloaded from $overrideUrl. Make sure you trust this source!',
emphasis: true,
);
_hasWarnedAboutStorageOverride = true;
}
/// Return the top-level directory in the cache; this is `bin/cache`.
Directory getRoot() {
if (_rootOverride != null) {
return _fileSystem.directory(_fileSystem.path.join(_rootOverride!.path, 'bin', 'cache'));
} else {
return _fileSystem.directory(_fileSystem.path.join(flutterRoot!, 'bin', 'cache'));
}
}
String getHostPlatformArchName() {
return getNameForHostPlatformArch(_osUtils.hostPlatform);
}
/// Return a directory in the cache dir. For `pkg`, this will return `bin/cache/pkg`.
Directory getCacheDir(String name) {
final Directory dir = _fileSystem.directory(_fileSystem.path.join(getRoot().path, name));
if (!dir.existsSync()) {
dir.createSync(recursive: true);
_osUtils.chmod(dir, '755');
}
return dir;
}
/// Return the top-level directory for artifact downloads.
Directory getDownloadDir() => getCacheDir('downloads');
/// Return the top-level mutable directory in the cache; this is `bin/cache/artifacts`.
Directory getCacheArtifacts() => getCacheDir('artifacts');
/// Location of LICENSE file.
File getLicenseFile() => _fileSystem.file(_fileSystem.path.join(flutterRoot!, 'LICENSE'));
/// Get a named directory from with the cache's artifact directory; for example,
/// `material_fonts` would return `bin/cache/artifacts/material_fonts`.
Directory getArtifactDirectory(String name) {
return getCacheArtifacts().childDirectory(name);
}
MapEntry<String, String> get dyLdLibEntry {
if (_dyLdLibEntry != null) {
return _dyLdLibEntry!;
}
final List<String> paths = <String>[];
for (final ArtifactSet artifact in _artifacts) {
final Map<String, String> env = artifact.environment;
if (env == null || !env.containsKey('DYLD_LIBRARY_PATH')) {
continue;
}
final String path = env['DYLD_LIBRARY_PATH']!;
if (path.isEmpty) {
continue;
}
paths.add(path);
}
_dyLdLibEntry = MapEntry<String, String>('DYLD_LIBRARY_PATH', paths.join(':'));
return _dyLdLibEntry!;
}
MapEntry<String, String>? _dyLdLibEntry;
/// The web sdk has to be co-located with the dart-sdk so that they can share source
/// code.
Directory getWebSdkDirectory() {
return getRoot().childDirectory('flutter_web_sdk');
}
String? getVersionFor(String artifactName) {
final File versionFile = _fileSystem.file(_fileSystem.path.join(
_rootOverride?.path ?? flutterRoot!,
'bin',
'internal',
'$artifactName.version',
));
return versionFile.existsSync() ? versionFile.readAsStringSync().trim() : null;
}
/// Delete all stamp files maintained by the cache.
void clearStampFiles() {
try {
getStampFileFor('flutter_tools').deleteSync();
for (final ArtifactSet artifact in _artifacts) {
final File file = getStampFileFor(artifact.stampName);
ErrorHandlingFileSystem.deleteIfExists(file);
}
} on FileSystemException catch (err) {
_logger.printError('Failed to delete some stamp files: $err');
}
}
/// Read the stamp for [artifactName].
///
/// If the file is missing or cannot be parsed, returns `null`.
String? getStampFor(String artifactName) {
final File stampFile = getStampFileFor(artifactName);
if (!stampFile.existsSync()) {
return null;
}
try {
return stampFile.readAsStringSync().trim();
} on FileSystemException {
return null;
}
}
void setStampFor(String artifactName, String version) {
getStampFileFor(artifactName).writeAsStringSync(version);
}
File getStampFileFor(String artifactName) {
return _fileSystem.file(_fileSystem.path.join(getRoot().path, '$artifactName.stamp'));
}
/// Returns `true` if either [entity] is older than the tools stamp or if
/// [entity] doesn't exist.
bool isOlderThanToolsStamp(FileSystemEntity entity) {
final File flutterToolsStamp = getStampFileFor('flutter_tools');
return _fsUtils.isOlderThanReference(
entity: entity,
referenceFile: flutterToolsStamp,
);
}
Future<bool> isUpToDate() async {
for (final ArtifactSet artifact in _artifacts) {
if (!await artifact.isUpToDate(_fileSystem)) {
return false;
}
}
return true;
}
/// Update the cache to contain all `requiredArtifacts`.
Future<void> updateAll(Set<DevelopmentArtifact> requiredArtifacts) async {
if (!_lockEnabled) {
return;
}
for (final ArtifactSet artifact in _artifacts) {
if (!requiredArtifacts.contains(artifact.developmentArtifact)) {
_logger.printTrace('Artifact $artifact is not required, skipping update.');
continue;
}
if (await artifact.isUpToDate(_fileSystem)) {
continue;
}
try {
await artifact.update(_artifactUpdater, _logger, _fileSystem, _osUtils);
} on SocketException catch (e) {
if (_hostsBlockedInChina.contains(e.address?.host)) {
_logger.printError(
'Failed to retrieve Flutter tool dependencies: ${e.message}.\n'
"If you're in China, please see this page: "
'https://flutter.dev/community/china',
emphasis: true,
);
}
rethrow;
}
}
}
Future<bool> areRemoteArtifactsAvailable({
String? engineVersion,
bool includeAllPlatforms = true,
}) async {
final bool includeAllPlatformsState = this.includeAllPlatforms;
bool allAvailable = true;
this.includeAllPlatforms = includeAllPlatforms;
for (final ArtifactSet cachedArtifact in _artifacts) {
if (cachedArtifact is EngineCachedArtifact) {
allAvailable &= await cachedArtifact.checkForArtifacts(engineVersion);
}
}
this.includeAllPlatforms = includeAllPlatformsState;
return allAvailable;
}
Future<bool> doesRemoteExist(String message, Uri url) async {
final Status status = _logger.startProgress(
message,
);
bool exists;
try {
exists = await _net.doesRemoteFileExist(url);
} finally {
status.stop();
}
return exists;
}
}
/// Representation of a set of artifacts used by the tool.
abstract class ArtifactSet {
ArtifactSet(this.developmentArtifact) : assert(developmentArtifact != null);
/// The development artifact.
final DevelopmentArtifact developmentArtifact;
/// [true] if the artifact is up to date.
Future<bool> isUpToDate(FileSystem fileSystem);
/// The environment variables (if any) required to consume the artifacts.
Map<String, String> get environment {
return const <String, String>{};
}
/// Updates the artifact.
Future<void> update(
ArtifactUpdater artifactUpdater,
Logger logger,
FileSystem fileSystem,
OperatingSystemUtils operatingSystemUtils,
);
/// The canonical name of the artifact.
String get name;
// The name of the stamp file. Defaults to the same as the
// artifact name.
String get stampName => name;
}
/// An artifact set managed by the cache.
abstract class CachedArtifact extends ArtifactSet {
CachedArtifact(
this.name,
this.cache,
DevelopmentArtifact developmentArtifact,
) : super(developmentArtifact);
final Cache cache;
@override
final String name;
@override
String get stampName => name;
Directory get location => cache.getArtifactDirectory(name);
String? get version => cache.getVersionFor(name);
// Whether or not to bypass normal platform filtering for this artifact.
bool get ignorePlatformFiltering {
return cache.includeAllPlatforms ||
(cache.platformOverrideArtifacts != null && cache.platformOverrideArtifacts!.contains(developmentArtifact.name));
}
@override
Future<bool> isUpToDate(FileSystem fileSystem) async {
if (!location.existsSync()) {
return false;
}
if (version != cache.getStampFor(stampName)) {
return false;
}
return isUpToDateInner(fileSystem);
}
@override
Future<void> update(
ArtifactUpdater artifactUpdater,
Logger logger,
FileSystem fileSystem,
OperatingSystemUtils operatingSystemUtils,
) async {
if (!location.existsSync()) {
try {
location.createSync(recursive: true);
} on FileSystemException catch (err) {
logger.printError(err.toString());
throwToolExit(
'Failed to create directory for flutter cache at ${location.path}. '
'Flutter may be missing permissions in its cache directory.'
);
}
}
await updateInner(artifactUpdater, fileSystem, operatingSystemUtils);
try {
if (version == null) {
logger.printError(
'No known version for the artifact name "$name". '
'Flutter can continue, but the artifact may be re-downloaded on '
'subsequent invocations until the problem is resolved.',
);
} else {
cache.setStampFor(stampName, version!);
}
} on FileSystemException catch (err) {
logger.printError(
'The new artifact "$name" was downloaded, but Flutter failed to update '
'its stamp file, receiving the error "$err". '
'Flutter can continue, but the artifact may be re-downloaded on '
'subsequent invocations until the problem is resolved.',
);
}
artifactUpdater.removeDownloadedFiles();
}
/// Hook method for extra checks for being up-to-date.
bool isUpToDateInner(FileSystem fileSystem) => true;
Future<void> updateInner(
ArtifactUpdater artifactUpdater,
FileSystem fileSystem,
OperatingSystemUtils operatingSystemUtils,
);
}
abstract class EngineCachedArtifact extends CachedArtifact {
EngineCachedArtifact(
this.stampName,
Cache cache,
DevelopmentArtifact developmentArtifact,
) : super('engine', cache, developmentArtifact);
@override
final String stampName;
/// Return a list of (directory path, download URL path) tuples.
List<List<String>> getBinaryDirs();
/// A list of cache directory paths to which the LICENSE file should be copied.
List<String> getLicenseDirs();
/// A list of the dart package directories to download.
List<String> getPackageDirs();
@override
bool isUpToDateInner(FileSystem fileSystem) {
final Directory pkgDir = cache.getCacheDir('pkg');
for (final String pkgName in getPackageDirs()) {
final String pkgPath = fileSystem.path.join(pkgDir.path, pkgName);
if (!fileSystem.directory(pkgPath).existsSync()) {
return false;
}
}
for (final List<String> toolsDir in getBinaryDirs()) {
final Directory dir = fileSystem.directory(fileSystem.path.join(location.path, toolsDir[0]));
if (!dir.existsSync()) {
return false;
}
}
for (final String licenseDir in getLicenseDirs()) {
final File file = fileSystem.file(fileSystem.path.join(location.path, licenseDir, 'LICENSE'));
if (!file.existsSync()) {
return false;
}
}
return true;
}
@override
Future<void> updateInner(
ArtifactUpdater artifactUpdater,
FileSystem fileSystem,
OperatingSystemUtils operatingSystemUtils,
) async {
final String url = '${cache.storageBaseUrl}/flutter_infra_release/flutter/$version/';
final Directory pkgDir = cache.getCacheDir('pkg');
for (final String pkgName in getPackageDirs()) {
await artifactUpdater.downloadZipArchive('Downloading package $pkgName...', Uri.parse('$url$pkgName.zip'), pkgDir);
}
for (final List<String> toolsDir in getBinaryDirs()) {
final String cacheDir = toolsDir[0];
final String urlPath = toolsDir[1];
final Directory dir = fileSystem.directory(fileSystem.path.join(location.path, cacheDir));
// Avoid printing things like 'Downloading linux-x64 tools...' multiple times.
final String friendlyName = urlPath.replaceAll('/artifacts.zip', '').replaceAll('.zip', '');
await artifactUpdater.downloadZipArchive('Downloading $friendlyName tools...', Uri.parse(url + urlPath), dir);
_makeFilesExecutable(dir, operatingSystemUtils);
final File frameworkZip = fileSystem.file(fileSystem.path.join(dir.path, 'FlutterMacOS.framework.zip'));
if (frameworkZip.existsSync()) {
final Directory framework = fileSystem.directory(fileSystem.path.join(dir.path, 'FlutterMacOS.framework'));
ErrorHandlingFileSystem.deleteIfExists(framework, recursive: true);
framework.createSync();
operatingSystemUtils.unzip(frameworkZip, framework);
}
}
final File licenseSource = cache.getLicenseFile();
for (final String licenseDir in getLicenseDirs()) {
final String licenseDestinationPath = fileSystem.path.join(location.path, licenseDir, 'LICENSE');
await licenseSource.copy(licenseDestinationPath);
}
}
Future<bool> checkForArtifacts(String? engineVersion) async {
engineVersion ??= version;
final String url = '${cache.storageBaseUrl}/flutter_infra_release/flutter/$engineVersion/';
bool exists = false;
for (final String pkgName in getPackageDirs()) {
exists = await cache.doesRemoteExist('Checking package $pkgName is available...', Uri.parse('$url$pkgName.zip'));
if (!exists) {
return false;
}
}
for (final List<String> toolsDir in getBinaryDirs()) {
final String cacheDir = toolsDir[0];
final String urlPath = toolsDir[1];
exists = await cache.doesRemoteExist('Checking $cacheDir tools are available...',
Uri.parse(url + urlPath));
if (!exists) {
return false;
}
}
return true;
}
void _makeFilesExecutable(Directory dir, OperatingSystemUtils operatingSystemUtils) {
operatingSystemUtils.chmod(dir, 'a+r,a+x');
for (final File file in dir.listSync(recursive: true).whereType<File>()) {
final FileStat stat = file.statSync();
final bool isUserExecutable = ((stat.mode >> 6) & 0x1) == 1;
if (file.basename == 'flutter_tester' || isUserExecutable) {
// Make the file readable and executable by all users.
operatingSystemUtils.chmod(file, 'a+r,a+x');
}
}
}
}
/// An API for downloading and un-archiving artifacts, such as engine binaries or
/// additional source code.
class ArtifactUpdater {
ArtifactUpdater({
required OperatingSystemUtils operatingSystemUtils,
required Logger logger,
required FileSystem fileSystem,
required Directory tempStorage,
required HttpClient httpClient,
required Platform platform,
}) : _operatingSystemUtils = operatingSystemUtils,
_httpClient = httpClient,
_logger = logger,
_fileSystem = fileSystem,
_tempStorage = tempStorage,
_platform = platform;
/// The number of times the artifact updater will repeat the artifact download loop.
static const int _kRetryCount = 2;
final Logger _logger;
final OperatingSystemUtils _operatingSystemUtils;
final FileSystem _fileSystem;
final Directory _tempStorage;
final HttpClient _httpClient;
final Platform _platform;
/// Keep track of the files we've downloaded for this execution so we
/// can delete them after completion. We don't delete them right after
/// extraction in case [update] is interrupted, so we can restart without
/// starting from scratch.
@visibleForTesting
final List<File> downloadedFiles = <File>[];
/// Download a zip archive from the given [url] and unzip it to [location].
Future<void> downloadZipArchive(
String message,
Uri url,
Directory location,
) {
return _downloadArchive(
message,
url,
location,
_operatingSystemUtils.unzip,
);
}
/// Download a gzipped tarball from the given [url] and unpack it to [location].
Future<void> downloadZippedTarball(String message, Uri url, Directory location) {
return _downloadArchive(
message,
url,
location,
_operatingSystemUtils.unpack,
);
}
/// Download an archive from the given [url] and unzip it to [location].
Future<void> _downloadArchive(
String message,
Uri url,
Directory location,
void Function(File, Directory) extractor,
) async {
final String downloadPath = flattenNameSubdirs(url, _fileSystem);
final File tempFile = _createDownloadFile(downloadPath);
Status status;
int retries = _kRetryCount;
while (retries > 0) {
status = _logger.startProgress(
message,
);
try {
_ensureExists(tempFile.parent);
if (tempFile.existsSync()) {
tempFile.deleteSync();
}
await _download(url, tempFile);
if (!tempFile.existsSync()) {
throw Exception('Did not find downloaded file ${tempFile.path}');
}
} on Exception catch (err) {
_logger.printTrace(err.toString());
retries -= 1;
if (retries == 0) {
throwToolExit(
'Failed to download $url. Ensure you have network connectivity and then try again.\n$err',
);
}
continue;
} on ArgumentError catch (error) {
final String? overrideUrl = _platform.environment['FLUTTER_STORAGE_BASE_URL'];
if (overrideUrl != null && url.toString().contains(overrideUrl)) {
_logger.printError(error.toString());
throwToolExit(
'The value of FLUTTER_STORAGE_BASE_URL ($overrideUrl) could not be '
'parsed as a valid url. Please see https://flutter.dev/community/china '
'for an example of how to use it.\n'
'Full URL: $url',
exitCode: kNetworkProblemExitCode,
);
}
// This error should not be hit if there was not a storage URL override, allow the
// tool to crash.
rethrow;
} finally {
status.stop();
}
/// Unzipping multiple file into a directory will not remove old files
/// from previous versions that are not present in the new bundle.
final Directory destination = location.childDirectory(
tempFile.fileSystem.path.basenameWithoutExtension(tempFile.path)
);
try {
ErrorHandlingFileSystem.deleteIfExists(
destination,
recursive: true,
);
} on FileSystemException catch (error) {
// Error that indicates another program has this file open and that it
// cannot be deleted. For the cache, this is either the analyzer reading
// the sky_engine package or a running flutter_tester device.
const int kSharingViolation = 32;
if (_platform.isWindows && error.osError?.errorCode == kSharingViolation) {
throwToolExit(
'Failed to delete ${destination.path} because the local file/directory is in use '
'by another process. Try closing any running IDEs or editors and trying '
'again'
);
}
}
_ensureExists(location);
try {
extractor(tempFile, location);
} on Exception catch (err) {
retries -= 1;
if (retries == 0) {
throwToolExit(
'Flutter could not download and/or extract $url. Ensure you have '
'network connectivity and all of the required dependencies listed at '
'flutter.dev/setup.\nThe original exception was: $err.'
);
}
_deleteIgnoringErrors(tempFile);
continue;
}
return;
}
}
/// Download bytes from [url], throwing non-200 responses as an exception.
///
/// Validates that the md5 of the content bytes matches the provided
/// `x-goog-hash` header, if present. This header should contain an md5 hash
/// if the download source is Google cloud storage.
///
/// See also:
/// * https://cloud.google.com/storage/docs/xml-api/reference-headers#xgooghash
Future<void> _download(Uri url, File file) async {
final HttpClientRequest request = await _httpClient.getUrl(url);
final HttpClientResponse response = await request.close();
if (response.statusCode != HttpStatus.ok) {
throw Exception(response.statusCode);
}
final String? md5Hash = _expectedMd5(response.headers);
ByteConversionSink? inputSink;
late StreamController<Digest> digests;
if (md5Hash != null) {
_logger.printTrace('Content $url md5 hash: $md5Hash');
digests = StreamController<Digest>();
inputSink = md5.startChunkedConversion(digests);
}
final RandomAccessFile randomAccessFile = file.openSync(mode: FileMode.writeOnly);
await response.forEach((List<int> chunk) {
inputSink?.add(chunk);
randomAccessFile.writeFromSync(chunk);
});
randomAccessFile.closeSync();
if (inputSink != null) {
inputSink.close();
final Digest digest = await digests.stream.last;
final String rawDigest = base64.encode(digest.bytes);
if (rawDigest != md5Hash) {
throw Exception(
'Expected $url to have md5 checksum $md5Hash, but was $rawDigest. This '
'may indicate a problem with your connection to the Flutter backend servers. '
'Please re-try the download after confirming that your network connection is '
'stable.'
);
}
}
}
String? _expectedMd5(HttpHeaders httpHeaders) {
final List<String>? values = httpHeaders['x-goog-hash'];
if (values == null) {
return null;
}
String? rawMd5Hash;
for (final String value in values) {
if (value.startsWith('md5=')) {
rawMd5Hash = value;
break;
}
}
if (rawMd5Hash == null) {
return null;
}
final List<String> segments = rawMd5Hash.split('md5=');
if (segments.length < 2) {
return null;
}
final String md5Hash = segments[1];
if (md5Hash.isEmpty) {
return null;
}
return md5Hash;
}
/// Create a temporary file and invoke [onTemporaryFile] with the file as
/// argument, then add the temporary file to the [downloadedFiles].
File _createDownloadFile(String name) {
final File tempFile = _fileSystem.file(_fileSystem.path.join(_tempStorage.path, name));
downloadedFiles.add(tempFile);
return tempFile;
}
/// Create the given [directory] and parents, as necessary.
void _ensureExists(Directory directory) {
if (!directory.existsSync()) {
directory.createSync(recursive: true);
}
}
/// Clear any zip/gzip files downloaded.
void removeDownloadedFiles() {
for (final File file in downloadedFiles) {
if (!file.existsSync()) {
continue;
}
try {
file.deleteSync();
} on FileSystemException catch (e) {
_logger.printError('Failed to delete "${file.path}". Please delete manually. $e');
continue;
}
for (Directory directory = file.parent; directory.absolute.path != _tempStorage.absolute.path; directory = directory.parent) {
if (directory.listSync().isNotEmpty) {
break;
}
_deleteIgnoringErrors(directory);
}
}
}
static void _deleteIgnoringErrors(FileSystemEntity entity) {
if (!entity.existsSync()) {
return;
}
try {
entity.deleteSync();
} on FileSystemException {
// Ignore errors.
}
}
}
@visibleForTesting
String flattenNameSubdirs(Uri url, FileSystem fileSystem) {
final List<String> pieces = <String>[url.host, ...url.pathSegments];
final Iterable<String> convertedPieces = pieces.map<String>(_flattenNameNoSubdirs);
return fileSystem.path.joinAll(convertedPieces);
}
/// Given a name containing slashes, colons, and backslashes, expand it into
/// something that doesn't.
String _flattenNameNoSubdirs(String fileName) {
final List<int> replacedCodeUnits = <int>[
for (int codeUnit in fileName.codeUnits)
..._flattenNameSubstitutions[codeUnit] ?? <int>[codeUnit],
];
return String.fromCharCodes(replacedCodeUnits);
}
// Many characters are problematic in filenames, especially on Windows.
final Map<int, List<int>> _flattenNameSubstitutions = <int, List<int>>{
r'@'.codeUnitAt(0): '@@'.codeUnits,
r'/'.codeUnitAt(0): '@s@'.codeUnits,
r'\'.codeUnitAt(0): '@bs@'.codeUnits,
r':'.codeUnitAt(0): '@c@'.codeUnits,
r'%'.codeUnitAt(0): '@per@'.codeUnits,
r'*'.codeUnitAt(0): '@ast@'.codeUnits,
r'<'.codeUnitAt(0): '@lt@'.codeUnits,
r'>'.codeUnitAt(0): '@gt@'.codeUnits,
r'"'.codeUnitAt(0): '@q@'.codeUnits,
r'|'.codeUnitAt(0): '@pip@'.codeUnits,
r'?'.codeUnitAt(0): '@ques@'.codeUnits,
};