Teach Linux to use local engine (#31631)
diff --git a/packages/flutter_tools/bin/tool_backend.dart b/packages/flutter_tools/bin/tool_backend.dart new file mode 100644 index 0000000..e264ad3 --- /dev/null +++ b/packages/flutter_tools/bin/tool_backend.dart
@@ -0,0 +1,82 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; // ignore: dart_io_import. +import 'package:path/path.dart' as path; // ignore: package_path_import. + +/// Executes the required flutter tasks for a linux build. +Future<void> main(List<String> arguments) async { + final String targetPlatform = arguments[0]; + final String buildMode = arguments[1]; + + final String projectDirectory = Platform.environment['PROJECT_DIR']; + final bool verbose = Platform.environment['VERBOSE_SCRIPT_LOGGING'] != null; + final bool trackWidgetCreation = Platform.environment['TRACK_WIDGET_CREATION'] != null; + final String flutterTarget = Platform.environment['FLUTTER_TARGET'] ?? 'lib/main.dart'; + final String flutterEngine = Platform.environment['FLUTTER_ENGINE']; + final String localEngine = Platform.environment['LOCAL_ENGINE']; + final String flutterRoot = Platform.environment['FLUTTER_ROOT']; + + Directory.current = projectDirectory; + + if (localEngine != null && !localEngine.contains(buildMode)) { + stderr.write(''' +ERROR: Requested build with Flutter local engine at '$localEngine' +This engine is not compatible with FLUTTER_BUILD_MODE: '$buildMode'. +You can fix this by updating the LOCAL_ENGINE environment variable, or +by running: + flutter build linux --local-engine=host_$buildMode +or + flutter build linux --local-engine=host_${buildMode}_unopt +======================================================================== +'''); + exit(1); + } + + String cacheDirectory; + switch (targetPlatform) { + case 'linux-x64': + cacheDirectory = 'linux/flutter'; + break; + default: + stderr.write('Unsupported target platform $targetPlatform'); + exit(1); + } + + final String flutterExecutable = path.join( + flutterRoot, 'bin', Platform.isWindows ? 'flutter.bat' : 'flutter'); + final ProcessResult unpackResult = await Process.run( + flutterExecutable, + <String>[ + '--suppress-analytics', + if (verbose) '--verbose', + 'unpack', + '--target-platform=$targetPlatform', + '--cache-dir=$cacheDirectory', + if (flutterEngine != null) '--local-engine-src-path=$flutterEngine', + if (localEngine != null) '--local-engine=$localEngine', + ]); + if (unpackResult.exitCode != 0) { + stderr.write(unpackResult.stderr); + exit(1); + } + final ProcessResult buildResult = await Process.run( + flutterExecutable, + <String>[ + '--suppress-analytics', + if (verbose) '--verbose', + 'build', + 'bundle', + '--target=$flutterTarget', + '--target-platform=$targetPlatform', + if (trackWidgetCreation) '--track-widget-creation', + if (flutterEngine != null) '--local-engine-src-path=$flutterEngine', + if (localEngine != null) '--local-engine=$localEngine', + ]); + if (buildResult.exitCode != 0) { + stderr.write(buildResult.stderr); + exit(1); + } + exit(0); +}
diff --git a/packages/flutter_tools/bin/tool_backend.sh b/packages/flutter_tools/bin/tool_backend.sh new file mode 100755 index 0000000..81de8b4 --- /dev/null +++ b/packages/flutter_tools/bin/tool_backend.sh
@@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +readonly flutter_bin_dir="${FLUTTER_ROOT}/bin" +readonly dart_bin_dir="${flutter_bin_dir}/cache/dart-sdk/bin" + +exec "${dart_bin_dir}/dart" "${FLUTTER_ROOT}/packages/flutter_tools/bin/tool_backend.dart" "${@:1}"
diff --git a/packages/flutter_tools/lib/executable.dart b/packages/flutter_tools/lib/executable.dart index 2671761..1e75509 100644 --- a/packages/flutter_tools/lib/executable.dart +++ b/packages/flutter_tools/lib/executable.dart
@@ -37,6 +37,7 @@ import 'src/commands/shell_completion.dart'; import 'src/commands/test.dart'; import 'src/commands/train.dart'; +import 'src/commands/unpack.dart'; import 'src/commands/update_packages.dart'; import 'src/commands/upgrade.dart'; import 'src/commands/version.dart'; @@ -82,6 +83,7 @@ ShellCompletionCommand(), TestCommand(verboseHelp: verboseHelp), TrainingCommand(), + UnpackCommand(), UpdatePackagesCommand(hidden: !verboseHelp), UpgradeCommand(), VersionCommand(),
diff --git a/packages/flutter_tools/lib/src/commands/unpack.dart b/packages/flutter_tools/lib/src/commands/unpack.dart new file mode 100644 index 0000000..79b7950 --- /dev/null +++ b/packages/flutter_tools/lib/src/commands/unpack.dart
@@ -0,0 +1,308 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import '../artifacts.dart'; +import '../base/common.dart'; +import '../base/file_system.dart'; +import '../base/io.dart'; +import '../base/process_manager.dart'; +import '../build_info.dart'; +import '../cache.dart'; +import '../globals.dart'; +import '../runner/flutter_command.dart'; + +/// The directory in the Flutter cache for each platform's artifacts. +const Map<TargetPlatform, String> flutterArtifactPlatformDirectory = + <TargetPlatform, String>{ + TargetPlatform.linux_x64: 'linux-x64', + TargetPlatform.darwin_x64: 'darwin-x64', + TargetPlatform.windows_x64: 'windows-x64', +}; + +// TODO(jonahwilliams): this should come from a configuration in each build +// directory. +const Map<TargetPlatform, List<String>> artifactFilesByPlatform = <TargetPlatform, List<String>>{ + TargetPlatform.linux_x64: <String>[ + 'libflutter_linux.so', + 'flutter_export.h', + 'flutter_messenger.h', + 'flutter_plugin_registrar.h', + 'flutter_glfw.h', + 'icudtl.dat', + 'cpp_client_wrapper/', + ], + TargetPlatform.darwin_x64: <String>[ + 'FlutterMacOS.framework', + ], + TargetPlatform.windows_x64: <String>[ + 'flutter_windows.dll', + 'flutter_windows.dll.exp', + 'flutter_windows.dll.lib', + 'flutter_windows.dll.pdb', + 'flutter_export.h', + 'flutter_messenger.h', + 'flutter_plugin_registrar.h', + 'flutter_glfw.h', + 'icudtl.dat', + 'cpp_client_wrapper/', + ], +}; + +/// Copies desktop artifacts to local cache directories. +class UnpackCommand extends FlutterCommand { + UnpackCommand() { + argParser.addOption( + 'target-platform', + allowed: <String>['darwin-x64', 'linux-x64', 'windows-x64'], + ); + argParser.addOption('cache-dir', + help: 'Location to output platform specific artifacts.'); + } + + @override + String get description => 'unpack desktop artifacts'; + + @override + String get name => 'unpack'; + + @override + bool get hidden => true; + + @override + bool get isExperimental => true; + + @override + Future<FlutterCommandResult> runCommand() async { + final String targetName = argResults['target-platform']; + final String targetDirectory = argResults['cache-dir']; + if (!fs.directory(targetDirectory).existsSync()) { + fs.directory(targetDirectory).createSync(recursive: true); + } + final TargetPlatform targetPlatform = getTargetPlatformForName(targetName); + final ArtifactUnpacker flutterArtifactFetcher = ArtifactUnpacker(targetPlatform); + bool success = true; + if (artifacts is LocalEngineArtifacts) { + final LocalEngineArtifacts localEngineArtifacts = artifacts; + success = flutterArtifactFetcher.copyLocalBuildArtifacts( + localEngineArtifacts.engineOutPath, + targetDirectory, + ); + } else { + success = flutterArtifactFetcher.copyCachedArtifacts( + targetDirectory, + ); + } + if (!success) { + throwToolExit('Failed to unpack desktop artifacts.'); + } + return null; + } +} + +/// Manages the copying of cached or locally built Flutter artifacts, including +/// tracking the last-copied versions and updating only if necessary. +class ArtifactUnpacker { + /// Creates a new fetcher for the given configuration. + const ArtifactUnpacker(this.platform); + + /// The platform to copy artifacts for. + final TargetPlatform platform; + + /// Checks [targetDirectory] to see if artifacts have already been copied for + /// the current hash, and if not, copies the artifacts for [platform] from the + /// Flutter cache (after ensuring that the cache is present). + /// + /// Returns true if the artifacts were successfully copied, or were already + /// present with the correct hash. + bool copyCachedArtifacts(String targetDirectory) { + String cacheStamp; + switch (platform) { + case TargetPlatform.linux_x64: + cacheStamp = 'linux-sdk'; + break; + case TargetPlatform.windows_x64: + cacheStamp = 'windows-sdk'; + break; + case TargetPlatform.darwin_x64: + cacheStamp = 'macos-sdk'; + break; + default: + throwToolExit('Unsupported target platform: $platform'); + } + final String targetHash = + readHashFileIfPossible(Cache.instance.getStampFileFor(cacheStamp)); + if (targetHash == null) { + printError('Failed to find engine stamp file'); + return false; + } + + try { + final String currentHash = _lastCopiedHash(targetDirectory); + if (currentHash == null || targetHash != currentHash) { + // Copy them to the target directory. + final String flutterCacheDirectory = fs.path.join( + Cache.flutterRoot, + 'bin', + 'cache', + 'artifacts', + 'engine', + flutterArtifactPlatformDirectory[platform], + ); + if (!_copyArtifactFiles(flutterCacheDirectory, targetDirectory)) { + return false; + } + _setLastCopiedHash(targetDirectory, targetHash); + printTrace('Copied artifacts for version $targetHash.'); + } else { + printTrace('Artifacts for version $targetHash already present.'); + } + } catch (error, stackTrace) { + printError(stackTrace.toString()); + printError(error.toString()); + return false; + } + return true; + } + + /// Acts like [copyCachedArtifacts], replacing the artifacts and updating + /// the version stamp, except that it pulls the artifact from a local engine + /// build with the given [buildConfiguration] (e.g., host_debug_unopt) whose + /// checkout is rooted at [engineRoot]. + bool copyLocalBuildArtifacts(String buildOutput, String targetDirectory) { + if (!_copyArtifactFiles(buildOutput, targetDirectory)) { + return false; + } + + // Update the hash file to indicate that it's a local build, so that it's + // obvious where it came from. + _setLastCopiedHash(targetDirectory, 'local build: $buildOutput'); + + return true; + } + + /// Copies the artifact files for [platform] from [sourceDirectory] to + /// [targetDirectory]. + bool _copyArtifactFiles(String sourceDirectory, String targetDirectory) { + final List<String> artifactFiles = artifactFilesByPlatform[platform]; + if (artifactFiles == null) { + printError('Unsupported platform: $platform.'); + return false; + } + + try { + fs.directory(targetDirectory).createSync(recursive: true); + + // On macOS, delete the existing framework if any before copying in the + // new one, since it's a directory. On the other platforms, where files + // are just individual files, this isn't necessary since copying over + // existing files will do the right thing. + if (platform == TargetPlatform.darwin_x64) { + _copyMacOSFramework( + fs.path.join(sourceDirectory, artifactFiles[0]), targetDirectory); + } else { + for (final String entityName in artifactFiles) { + final String sourcePath = fs.path.join(sourceDirectory, entityName); + final String targetPath = fs.path.join(targetDirectory, entityName); + if (entityName.endsWith('/')) { + copyDirectorySync( + fs.directory(sourcePath), + fs.directory(targetPath), + ); + } else { + fs.file(sourcePath) + .copySync(fs.path.join(targetDirectory, entityName)); + } + } + } + + printTrace('Copied artifacts from $sourceDirectory.'); + } catch (e, stackTrace) { + printError(stackTrace.toString()); + printError(e.message); + return false; + } + return true; + } + + /// Returns a File object for the file containing the last copied hash + /// in [directory]. + File _lastCopiedHashFile(String directory) { + return fs.file(fs.path.join(directory, '.last_artifact_version')); + } + + /// Returns the hash of the artifacts last copied to [directory], or null if + /// they haven't been copied. + String _lastCopiedHash(String directory) { + // Sanity check that at least one file is present; this won't catch every + // case, but handles someone deleting all the non-hidden cached files to + // force fresh copy. + final String artifactFilePath = fs.path.join( + directory, + artifactFilesByPlatform[platform].first, + ); + if (!fs.file(artifactFilePath).existsSync()) { + return null; + } + final File hashFile = _lastCopiedHashFile(directory); + return readHashFileIfPossible(hashFile); + } + + /// Writes [hash] to the file that stores the last copied hash for + /// in [directory]. + void _setLastCopiedHash(String directory, String hash) { + _lastCopiedHashFile(directory).writeAsStringSync(hash); + } + + /// Copies the framework at [frameworkPath] to [targetDirectory] + /// by invoking 'cp -R'. + /// + /// The shelling out is done to avoid complications with preserving special + /// files (e.g., symbolic links) in the framework structure. + /// + /// Removes any previous version of the framework that already exists in the + /// target directory. + void _copyMacOSFramework(String frameworkPath, String targetDirectory) { + _deleteFrameworkIfPresent( + fs.path.join(targetDirectory, fs.path.basename(frameworkPath))); + + final ProcessResult result = processManager + .runSync(<String>['cp', '-R', frameworkPath, targetDirectory]); + if (result.exitCode != 0) { + throw Exception( + 'Failed to copy framework (exit ${result.exitCode}:\n' + '${result.stdout}\n---\n${result.stderr}', + ); + } + } + + /// Recursively deletes the framework at [frameworkPath], if it exists. + void _deleteFrameworkIfPresent(String frameworkPath) { + // Ensure that the path is a framework, to minimize the potential for + // catastrophic deletion bugs with bad arguments. + if (fs.path.extension(frameworkPath) != '.framework') { + throw Exception( + 'Attempted to delete a non-framework directory: $frameworkPath'); + } + + final Directory directory = fs.directory(frameworkPath); + if (directory.existsSync()) { + directory.deleteSync(recursive: true); + } + } + + /// Returns the engine hash from [file] as a String, or null. + /// + /// If the file is missing, or cannot be read, returns null. + String readHashFileIfPossible(File file) { + if (!file.existsSync()) { + return null; + } + try { + return file.readAsStringSync().trim(); + } on FileSystemException { + // If the file can't be read for any reason, just treat it as missing. + return null; + } + } +}
diff --git a/packages/flutter_tools/lib/src/linux/build_linux.dart b/packages/flutter_tools/lib/src/linux/build_linux.dart index c71d11a..9010095 100644 --- a/packages/flutter_tools/lib/src/linux/build_linux.dart +++ b/packages/flutter_tools/lib/src/linux/build_linux.dart
@@ -2,7 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import '../artifacts.dart'; import '../base/common.dart'; +import '../base/file_system.dart'; import '../base/io.dart'; import '../base/logger.dart'; import '../base/process_manager.dart'; @@ -13,21 +15,32 @@ import '../project.dart'; /// Builds the Linux project through the Makefile. -Future<void> buildLinux(LinuxProject linuxProject, BuildInfo buildInfo) async { - /// Cache flutter root in linux directory. - linuxProject.editableHostAppDirectory.childFile('.generated_flutter_root') - ..createSync(recursive: true) - ..writeAsStringSync(Cache.flutterRoot); - +Future<void> buildLinux(LinuxProject linuxProject, BuildInfo buildInfo, {String target = 'lib/main.dart'}) async { final String buildFlag = buildInfo?.isDebug == true ? 'debug' : 'release'; - final String bundleFlags = buildInfo?.trackWidgetCreation == true ? '--track-widget-creation' : ''; + final StringBuffer buffer = StringBuffer(''' +# Generated code do not commit. +export FLUTTER_ROOT=${Cache.flutterRoot} +export BUILD=$buildFlag +export TRACK_WIDGET_CREATION=${buildInfo?.trackWidgetCreation == true} +export FLUTTER_TARGET=$target +export PROJECT_DIR=${linuxProject.project.directory.path} +'''); + if (artifacts is LocalEngineArtifacts) { + final LocalEngineArtifacts localEngineArtifacts = artifacts; + final String engineOutPath = localEngineArtifacts.engineOutPath; + buffer.writeln('export FLUTTER_ENGINE=${fs.path.dirname(fs.path.dirname(engineOutPath))}'); + buffer.writeln('export LOCAL_ENGINE=${fs.path.basename(engineOutPath)}'); + } + + /// Cache flutter configuration files in the linux directory. + linuxProject.cacheDirectory.childFile('generated_config') + ..createSync(recursive: true) + ..writeAsStringSync(buffer.toString()); + final Process process = await processManager.start(<String>[ 'make', '-C', linuxProject.editableHostAppDirectory.path, - 'BUILD=$buildFlag', - 'FLUTTER_ROOT=${Cache.flutterRoot}', - 'FLUTTER_BUNDLE_FLAGS=$bundleFlags', ], runInShell: true); final Status status = logger.startProgress( 'Building Linux application...',
diff --git a/packages/flutter_tools/lib/src/project.dart b/packages/flutter_tools/lib/src/project.dart index 5891c6c..ae8e405 100644 --- a/packages/flutter_tools/lib/src/project.dart +++ b/packages/flutter_tools/lib/src/project.dart
@@ -599,6 +599,8 @@ Directory get editableHostAppDirectory => project.directory.childDirectory('linux'); + Directory get cacheDirectory => editableHostAppDirectory.childDirectory('flutter'); + bool existsSync() => editableHostAppDirectory.existsSync(); /// The Linux project makefile.
diff --git a/packages/flutter_tools/pubspec.yaml b/packages/flutter_tools/pubspec.yaml index 5d0fd99..fd65f63 100644 --- a/packages/flutter_tools/pubspec.yaml +++ b/packages/flutter_tools/pubspec.yaml
@@ -5,7 +5,7 @@ environment: # The pub client defaults to an <2.0.0 sdk constraint which we need to explicitly overwrite. - sdk: ">=2.2.0 <3.0.0" + sdk: ">=2.2.2 <3.0.0" dependencies: # To update these, use "flutter update-packages --force-upgrade".
diff --git a/packages/flutter_tools/test/commands/build_linux_test.dart b/packages/flutter_tools/test/commands/build_linux_test.dart index 146b438..3e59b24 100644 --- a/packages/flutter_tools/test/commands/build_linux_test.dart +++ b/packages/flutter_tools/test/commands/build_linux_test.dart
@@ -21,7 +21,6 @@ void main() { Cache.disableLocking(); final MockProcessManager mockProcessManager = MockProcessManager(); - final MemoryFileSystem memoryFilesystem = MemoryFileSystem(); final MockProcess mockProcess = MockProcess(); final MockPlatform linuxPlatform = MockPlatform(); final MockPlatform notLinuxPlatform = MockPlatform(); @@ -46,7 +45,7 @@ ), throwsA(isInstanceOf<ToolExit>())); }, overrides: <Type, Generator>{ Platform: () => linuxPlatform, - FileSystem: () => memoryFilesystem, + FileSystem: () => MemoryFileSystem(), }); testUsingContext('Linux build fails on non-linux platform', () async { @@ -61,22 +60,20 @@ ), throwsA(isInstanceOf<ToolExit>())); }, overrides: <Type, Generator>{ Platform: () => notLinuxPlatform, - FileSystem: () => memoryFilesystem, + FileSystem: () => MemoryFileSystem(), }); - testUsingContext('Linux build invokes make', () async { + testUsingContext('Linux build invokes make and writes temporary files', () async { final BuildCommand command = BuildCommand(); applyMocksToCommand(command); fs.file('linux/build.sh').createSync(recursive: true); fs.file('pubspec.yaml').createSync(); fs.file('.packages').createSync(); + when(mockProcessManager.start(<String>[ 'make', '-C', '/linux', - 'BUILD=release', - 'FLUTTER_ROOT=/', - 'FLUTTER_BUNDLE_FLAGS=', ], runInShell: true)).thenAnswer((Invocation invocation) async { return mockProcess; }); @@ -84,8 +81,9 @@ await createTestCommandRunner(command).run( const <String>['build', 'linux'] ); + expect(fs.file('linux/flutter/generated_config').existsSync(), true); }, overrides: <Type, Generator>{ - FileSystem: () => memoryFilesystem, + FileSystem: () => MemoryFileSystem(), ProcessManager: () => mockProcessManager, Platform: () => linuxPlatform, });