lib/web_ui/dev/test_runner.dart - mirrors/engine - Git at Google

 // Copyright 2013 The Flutter Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 import 'dart:async';
 import 'dart:io' as io;

 import 'package:args/command_runner.dart';
 import 'package:path/path.dart' as path;

 import 'package:watcher/src/watch_event.dart';

 import 'environment.dart';
 import 'exceptions.dart';
 import 'felt_config.dart';
 import 'generate_builder_json.dart';
 import 'pipeline.dart';
 import 'steps/compile_bundle_step.dart';
 import 'steps/copy_artifacts_step.dart';
 import 'steps/run_suite_step.dart';
 import 'suite_filter.dart';
 import 'utils.dart';

 /// Runs tests.
 class TestCommand extends Command<bool> with ArgUtils<bool> {
   TestCommand() {
     argParser
       ..addFlag(
         'start-paused',
         help: 'Pauses the browser before running a test, giving you an '
             'opportunity to add breakpoints or inspect loaded code before '
             'running the code.',
       )
       ..addFlag(
         'verbose',
         abbr: 'v',
         help: 'Enable verbose output.'
       )
       ..addFlag(
         'watch',
         abbr: 'w',
         help: 'Run in watch mode so the tests re-run whenever a change is '
             'made.',
       )
       ..addFlag(
         'list',
         help:
             'Lists the bundles that would be compiled and the suites that '
             'will be run as part of this invocation, without actually '
             'compiling or running them.'
       )
       ..addFlag(
         'generate-builder-json',
         help:
             'Generates JSON for the engine_v2 builders to build and copy all'
             'artifacts, compile all test bundles, and run all test suites on'
             'all platforms.'
       )
       ..addFlag(
         'compile',
         help:
             'Compile test bundles. If this is specified on its own, we will '
             'only compile and not run the suites.'
       )
       ..addFlag(
         'run',
         help:
             'Run test suites. If this is specified on its own, we will only '
             'run the suites and not compile the bundles.'
       )
       ..addFlag(
         'copy-artifacts',
         help:
             'Copy artifacts needed for test suites. If this is specified on '
             'its own, we will only copy the artifacts and not compile or run'
             'the tests bundles or suites.'
       )
       ..addFlag(
         'profile',
         help:
             'Use artifacts from the profile build instead of release.'
       )
       ..addFlag(
         'debug',
         help: 'Use artifacts from the debug build instead of release.'
       )
       ..addFlag(
         'dwarf',
         help: 'Debug wasm modules using embedded DWARF data.'
       )
       ..addFlag(
         'require-skia-gold',
         help:
             'Whether we require Skia Gold to be available or not. When this '
             'flag is true, the tests will fail if Skia Gold is not available.',
       )
       ..addFlag(
         'update-screenshot-goldens',
         help:
             'When running screenshot tests writes them to the file system into '
             '.dart_tool/goldens. Use this option to bulk-update all screenshots, '
             'for example, when a new browser version affects pixels.',
       )
       ..addMultiOption(
         'browser',
         help: 'Filter test suites by browser.',
       )
       ..addMultiOption(
         'compiler',
         help: 'Filter test suites by compiler.',
       )
       ..addMultiOption(
         'renderer',
         help: 'Filter test suites by renderer.',
       )
       ..addMultiOption(
         'canvaskit-variant',
         help: 'Filter test suites by CanvasKit variant.',
       )
       ..addMultiOption(
         'suite',
         help: 'Filter test suites by suite name.',
       )
       ..addMultiOption(
         'bundle',
         help: 'Filter test suites by bundle name.',
       )
       ..addFlag(
         'fail-early',
         help: 'If set, causes the test runner to exit upon the first test '
               'failure. If not set, the test runner will continue running '
               'test despite failures and will report them after all tests '
               'finish.',
       )
       ..addOption(
         'canvaskit-path',
         help: 'Optional. The path to a local build of CanvasKit to use in '
               'tests. If omitted, the test runner uses the default CanvasKit '
               'build.',
       )
       ..addFlag(
         'wasm',
         help: 'Whether the test we are running are compiled to webassembly.'
       );
   }

   @override
   final String name = 'test';

   @override
   final String description = 'Run tests.';

   bool get isWatchMode => boolArg('watch');

   bool get isList => boolArg('list');

   bool get failEarly => boolArg('fail-early');

   /// Whether to start the browser in debug mode.
   ///
   /// In this mode the browser pauses before running the test to allow
   /// you set breakpoints or inspect the code.
   bool get startPaused => boolArg('start-paused');

   bool get isVerbose => boolArg('verbose');

   /// The target test files to run.
   List<FilePath> get targetFiles => argResults!.rest.map((String t) => FilePath.fromCwd(t)).toList();

   /// When running screenshot tests, require Skia Gold to be available and
   /// reachable.
   bool get requireSkiaGold => boolArg('require-skia-gold');

   /// When running screenshot tests writes them to the file system into
   /// ".dart_tool/goldens".
   bool get doUpdateScreenshotGoldens => boolArg('update-screenshot-goldens');

   /// Path to a CanvasKit build. Overrides the default CanvasKit.
   String? get overridePathToCanvasKit => argResults!['canvaskit-path'] as String?;

   final FeltConfig config = FeltConfig.fromFile(
     path.join(environment.webUiTestDir.path, 'felt_config.yaml')
   );

   BrowserSuiteFilter? makeBrowserFilter() {
     final List<String>? browserArgs = argResults!['browser'] as List<String>?;
     if (browserArgs == null || browserArgs.isEmpty) {
       return null;
     }
     final Set<BrowserName> browserNames = Set<BrowserName>.from(browserArgs.map((String arg) => BrowserName.values.byName(arg)));
     return BrowserSuiteFilter(allowList: browserNames);
   }

   CompilerFilter? makeCompilerFilter() {
     final List<String>? compilerArgs = argResults!['compiler'] as List<String>?;
     if (compilerArgs == null || compilerArgs.isEmpty) {
       return null;
     }
     final Set<Compiler> compilers = Set<Compiler>.from(compilerArgs.map((String arg) => Compiler.values.byName(arg)));
     return CompilerFilter(allowList: compilers);
   }

   RendererFilter? makeRendererFilter() {
     final List<String>? rendererArgs = argResults!['renderer'] as List<String>?;
     if (rendererArgs == null || rendererArgs.isEmpty) {
       return null;
     }
     final Set<Renderer> renderers = Set<Renderer>.from(rendererArgs.map((String arg) => Renderer.values.byName(arg)));
     return RendererFilter(allowList: renderers);
   }

   CanvasKitVariantFilter? makeCanvasKitVariantFilter() {
     final List<String>? variantArgs = argResults!['canvaskit-variant'] as List<String>?;
     if (variantArgs == null || variantArgs.isEmpty) {
       return null;
     }
     final Set<CanvasKitVariant> variants = Set<CanvasKitVariant>.from(variantArgs.map((String arg) => CanvasKitVariant.values.byName(arg)));
     return CanvasKitVariantFilter(allowList: variants);
   }

   SuiteNameFilter? makeSuiteNameFilter() {
     final List<String>? suiteNameArgs = argResults!['suite'] as List<String>?;
     if (suiteNameArgs == null || suiteNameArgs.isEmpty) {
       return null;
     }

     final Iterable<String> allSuiteNames = config.testSuites.map((TestSuite suite) => suite.name);
     for (final String suiteName in suiteNameArgs) {
       if (!allSuiteNames.contains(suiteName)) {
         throw ToolExit('No suite found named $suiteName');
       }
     }
     return SuiteNameFilter(allowList: Set<String>.from(suiteNameArgs));
   }

   BundleNameFilter? makeBundleNameFilter() {
     final List<String>? bundleNameArgs = argResults!['bundle'] as List<String>?;
     if (bundleNameArgs == null || bundleNameArgs.isEmpty) {
       return null;
     }

     final Iterable<String> allBundleNames = config.testSuites.map(
       (TestSuite suite) => suite.testBundle.name
     );
     for (final String bundleName in bundleNameArgs) {
       if (!allBundleNames.contains(bundleName)) {
         throw ToolExit('No bundle found named $bundleName');
       }
     }
     return BundleNameFilter(allowList: Set<String>.from(bundleNameArgs));
   }

   FileFilter? makeFileFilter() {
     final List<FilePath> tests = targetFiles;
     if (tests.isEmpty) {
       return null;
     }
     final Set<String> bundleNames = <String>{};
     for (final FilePath testPath in tests) {
       if (!io.File(testPath.absolute).existsSync()) {
         throw ToolExit('Test path not found: $testPath');
       }
       bool bundleFound = false;
       for (final TestBundle bundle in config.testBundles) {
         final String testSetPath = getTestSetDirectory(bundle.testSet).path;
         if (path.isWithin(testSetPath, testPath.absolute)) {
           bundleFound = true;
           bundleNames.add(bundle.name);
         }
       }
       if (!bundleFound) {
         throw ToolExit('Test path not in any known test bundle: $testPath');
       }
     }
     return FileFilter(allowList: bundleNames);
   }

   List<SuiteFilter> get suiteFilters {
     final BrowserSuiteFilter? browserFilter = makeBrowserFilter();
     final CompilerFilter? compilerFilter = makeCompilerFilter();
     final RendererFilter? rendererFilter = makeRendererFilter();
     final CanvasKitVariantFilter? canvaskitVariantFilter = makeCanvasKitVariantFilter();
     final SuiteNameFilter? suiteNameFilter = makeSuiteNameFilter();
     final BundleNameFilter? bundleNameFilter = makeBundleNameFilter();
     final FileFilter? fileFilter = makeFileFilter();
     return <SuiteFilter>[
       PlatformBrowserFilter(),
       if (browserFilter != null) browserFilter,
       if (compilerFilter != null) compilerFilter,
       if (rendererFilter != null) rendererFilter,
       if (canvaskitVariantFilter != null) canvaskitVariantFilter,
       if (suiteNameFilter != null) suiteNameFilter,
       if (bundleNameFilter != null) bundleNameFilter,
       if (fileFilter != null) fileFilter,
     ];
   }

   List<TestSuite> _filterTestSuites() {
     if (isVerbose) {
       print('Filtering suites...');
     }
     final List<SuiteFilter> filters = suiteFilters;
     final List<TestSuite> filteredSuites = config.testSuites.where((TestSuite suite) {
       for (final SuiteFilter filter in filters) {
         final SuiteFilterResult result = filter.filterSuite(suite);
         if (!result.isAccepted) {
           if (isVerbose) {
             print('  ${suite.name.ansiCyan} rejected for reason: ${result.rejectReason}');
           }
           return false;
         }
       }
       return true;
     }).toList();
     return filteredSuites;
   }

   List<TestBundle> _filterBundlesForSuites(List<TestSuite> suites) {
     final Set<TestBundle> seenBundles =
       Set<TestBundle>.from(suites.map((TestSuite suite) => suite.testBundle));
     return config.testBundles.where((TestBundle bundle) => seenBundles.contains(bundle)).toList();
   }

   ArtifactDependencies _artifactsForSuites(List<TestSuite> suites) {
     return suites.fold(ArtifactDependencies.none(),
       (ArtifactDependencies deps, TestSuite suite) => deps | suite.artifactDependencies);
   }

   @override
   Future<bool> run() async {
     final List<TestSuite> filteredSuites = _filterTestSuites();
     final List<TestBundle> bundles = _filterBundlesForSuites(filteredSuites);
     final ArtifactDependencies artifacts = _artifactsForSuites(filteredSuites);
     if (boolArg('generate-builder-json')) {
       final String configString = generateBuilderJson(config);
       final io.File configFile = io.File(path.join(
         environment.flutterDirectory.path,
         'ci',
         'builders',
         'linux_web_engine.json',
       ));
       configFile.writeAsStringSync(configString);
       return true;
     }
     if (isList || isVerbose) {
       print('Suites:');
       for (final TestSuite suite in filteredSuites) {
         print('  ${suite.name.ansiCyan}');
       }
       print('Bundles:');
       for (final TestBundle bundle in bundles) {
         print('  ${bundle.name.ansiMagenta}');
       }
       print('Artifacts:');
       if (artifacts.canvasKit) {
         print('  canvaskit'.ansiYellow);
       }
       if (artifacts.canvasKitChromium) {
         print('  canvaskit_chromium'.ansiYellow);
       }
       if (artifacts.skwasm) {
         print('  skwasm'.ansiYellow);
       }
     }
     if (isList) {
       return true;
     }

     bool shouldRun = boolArg('run');
     bool shouldCompile = boolArg('compile');
     bool shouldCopyArtifacts = boolArg('copy-artifacts');
     if (!shouldRun && !shouldCompile && !shouldCopyArtifacts) {
       // If none of these is specified, we should assume we need to do all of them.
       shouldRun = true;
       shouldCompile = true;
       shouldCopyArtifacts = true;
     }

     final Set<FilePath>? testFiles = targetFiles.isEmpty ? null : Set<FilePath>.from(targetFiles);
     final Pipeline testPipeline = Pipeline(steps: <PipelineStep>[
       if (isWatchMode) ClearTerminalScreenStep(),
       if (shouldCopyArtifacts) CopyArtifactsStep(artifacts, runtimeMode: runtimeMode),
       if (shouldCompile)
         for (final TestBundle bundle in bundles)
           CompileBundleStep(
             bundle: bundle,
             isVerbose: isVerbose,
             testFiles: testFiles,
           ),
       if (shouldRun)
         for (final TestSuite suite in filteredSuites)
           RunSuiteStep(
             suite,
             startPaused: startPaused,
             isVerbose: isVerbose,
             doUpdateScreenshotGoldens: doUpdateScreenshotGoldens,
             requireSkiaGold: requireSkiaGold,
             overridePathToCanvasKit: overridePathToCanvasKit,
             testFiles: testFiles,
             useDwarf: boolArg('dwarf'),
           ),
     ]);

     try {
       await testPipeline.run();
       if (isWatchMode) {
         print('');
         print('Initial test succeeded!');
       }
     } catch(error, stackTrace) {
       if (isWatchMode) {
         // The error is printed but not rethrown in watch mode because
         // failures are expected. The idea is that the developer corrects the
         // error, saves the file, and the pipeline reruns.
         print('');
         print('Initial test failed!\n');
         print(error);
         print(stackTrace);
       } else {
         rethrow;
       }
     }

     if (isWatchMode) {
       final FilePath dir = FilePath.fromWebUi('');
       print('');
       print(
           'Watching ${dir.relativeToCwd}/lib and ${dir.relativeToCwd}/test to re-run tests');
       print('');
       await PipelineWatcher(
           dir: dir.absolute,
           pipeline: testPipeline,
           ignore: (WatchEvent event) {
             // Ignore font files that are copied whenever tests run.
             if (event.path.endsWith('.ttf')) {
               return true;
             }

             // React to changes in lib/ and test/ folders.
             final String relativePath =
                 path.relative(event.path, from: dir.absolute);
             if (path.isWithin('lib', relativePath) ||
                 path.isWithin('test', relativePath)) {
               return false;
             }

             // Ignore anything else.
             return true;
           }).start();
     }
     return true;
   }
 }

 /// Clears the terminal screen and places the cursor at the top left corner.
 ///
 /// This works on Linux and Mac. On Windows, it's a no-op.
 class ClearTerminalScreenStep implements PipelineStep {
   @override
   String get description => 'clearing terminal screen';

   @override
   bool get isSafeToInterrupt => false;

   @override
   Future<void> interrupt() async {}

   @override
   Future<void> run() async {
     if (!io.Platform.isWindows) {
       // See: https://en.wikipedia.org/wiki/ANSI_escape_code#CSI_sequences
       print('\x1B[2J\x1B[1;2H');
     }
   }
 }
	// Copyright 2013 The Flutter Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	import 'dart:async';
	import 'dart:io' as io;

	import 'package:args/command_runner.dart';
	import 'package:path/path.dart' as path;

	import 'package:watcher/src/watch_event.dart';

	import 'environment.dart';
	import 'exceptions.dart';
	import 'felt_config.dart';
	import 'generate_builder_json.dart';
	import 'pipeline.dart';
	import 'steps/compile_bundle_step.dart';
	import 'steps/copy_artifacts_step.dart';
	import 'steps/run_suite_step.dart';
	import 'suite_filter.dart';
	import 'utils.dart';

	/// Runs tests.
	class TestCommand extends Command<bool> with ArgUtils<bool> {
	TestCommand() {
	argParser
	..addFlag(
	'start-paused',
	help: 'Pauses the browser before running a test, giving you an '
	'opportunity to add breakpoints or inspect loaded code before '
	'running the code.',
	)
	..addFlag(
	'verbose',
	abbr: 'v',
	help: 'Enable verbose output.'
	)
	..addFlag(
	'watch',
	abbr: 'w',
	help: 'Run in watch mode so the tests re-run whenever a change is '
	'made.',
	)
	..addFlag(
	'list',
	help:
	'Lists the bundles that would be compiled and the suites that '
	'will be run as part of this invocation, without actually '
	'compiling or running them.'
	)
	..addFlag(
	'generate-builder-json',
	help:
	'Generates JSON for the engine_v2 builders to build and copy all'
	'artifacts, compile all test bundles, and run all test suites on'
	'all platforms.'
	)
	..addFlag(
	'compile',
	help:
	'Compile test bundles. If this is specified on its own, we will '
	'only compile and not run the suites.'
	)
	..addFlag(
	'run',
	help:
	'Run test suites. If this is specified on its own, we will only '
	'run the suites and not compile the bundles.'
	)
	..addFlag(
	'copy-artifacts',
	help:
	'Copy artifacts needed for test suites. If this is specified on '
	'its own, we will only copy the artifacts and not compile or run'
	'the tests bundles or suites.'
	)
	..addFlag(
	'profile',
	help:
	'Use artifacts from the profile build instead of release.'
	)
	..addFlag(
	'debug',
	help: 'Use artifacts from the debug build instead of release.'
	)
	..addFlag(
	'dwarf',
	help: 'Debug wasm modules using embedded DWARF data.'
	)
	..addFlag(
	'require-skia-gold',
	help:
	'Whether we require Skia Gold to be available or not. When this '
	'flag is true, the tests will fail if Skia Gold is not available.',
	)
	..addFlag(
	'update-screenshot-goldens',
	help:
	'When running screenshot tests writes them to the file system into '
	'.dart_tool/goldens. Use this option to bulk-update all screenshots, '
	'for example, when a new browser version affects pixels.',
	)
	..addMultiOption(
	'browser',
	help: 'Filter test suites by browser.',
	)
	..addMultiOption(
	'compiler',
	help: 'Filter test suites by compiler.',
	)
	..addMultiOption(
	'renderer',
	help: 'Filter test suites by renderer.',
	)
	..addMultiOption(
	'canvaskit-variant',
	help: 'Filter test suites by CanvasKit variant.',
	)
	..addMultiOption(
	'suite',
	help: 'Filter test suites by suite name.',
	)
	..addMultiOption(
	'bundle',
	help: 'Filter test suites by bundle name.',
	)
	..addFlag(
	'fail-early',
	help: 'If set, causes the test runner to exit upon the first test '
	'failure. If not set, the test runner will continue running '
	'test despite failures and will report them after all tests '
	'finish.',
	)
	..addOption(
	'canvaskit-path',
	help: 'Optional. The path to a local build of CanvasKit to use in '
	'tests. If omitted, the test runner uses the default CanvasKit '
	'build.',
	)
	..addFlag(
	'wasm',
	help: 'Whether the test we are running are compiled to webassembly.'
	);
	}

	@override
	final String name = 'test';

	@override
	final String description = 'Run tests.';

	bool get isWatchMode => boolArg('watch');

	bool get isList => boolArg('list');

	bool get failEarly => boolArg('fail-early');

	/// Whether to start the browser in debug mode.
	///
	/// In this mode the browser pauses before running the test to allow
	/// you set breakpoints or inspect the code.
	bool get startPaused => boolArg('start-paused');

	bool get isVerbose => boolArg('verbose');

	/// The target test files to run.
	List<FilePath> get targetFiles => argResults!.rest.map((String t) => FilePath.fromCwd(t)).toList();

	/// When running screenshot tests, require Skia Gold to be available and
	/// reachable.
	bool get requireSkiaGold => boolArg('require-skia-gold');

	/// When running screenshot tests writes them to the file system into
	/// ".dart_tool/goldens".
	bool get doUpdateScreenshotGoldens => boolArg('update-screenshot-goldens');

	/// Path to a CanvasKit build. Overrides the default CanvasKit.
	String? get overridePathToCanvasKit => argResults!['canvaskit-path'] as String?;

	final FeltConfig config = FeltConfig.fromFile(
	path.join(environment.webUiTestDir.path, 'felt_config.yaml')
	);

	BrowserSuiteFilter? makeBrowserFilter() {
	final List<String>? browserArgs = argResults!['browser'] as List<String>?;
	if (browserArgs == null \|\| browserArgs.isEmpty) {
	return null;
	}
	final Set<BrowserName> browserNames = Set<BrowserName>.from(browserArgs.map((String arg) => BrowserName.values.byName(arg)));
	return BrowserSuiteFilter(allowList: browserNames);
	}

	CompilerFilter? makeCompilerFilter() {
	final List<String>? compilerArgs = argResults!['compiler'] as List<String>?;
	if (compilerArgs == null \|\| compilerArgs.isEmpty) {
	return null;
	}
	final Set<Compiler> compilers = Set<Compiler>.from(compilerArgs.map((String arg) => Compiler.values.byName(arg)));
	return CompilerFilter(allowList: compilers);
	}

	RendererFilter? makeRendererFilter() {
	final List<String>? rendererArgs = argResults!['renderer'] as List<String>?;
	if (rendererArgs == null \|\| rendererArgs.isEmpty) {
	return null;
	}
	final Set<Renderer> renderers = Set<Renderer>.from(rendererArgs.map((String arg) => Renderer.values.byName(arg)));
	return RendererFilter(allowList: renderers);
	}

	CanvasKitVariantFilter? makeCanvasKitVariantFilter() {
	final List<String>? variantArgs = argResults!['canvaskit-variant'] as List<String>?;
	if (variantArgs == null \|\| variantArgs.isEmpty) {
	return null;
	}
	final Set<CanvasKitVariant> variants = Set<CanvasKitVariant>.from(variantArgs.map((String arg) => CanvasKitVariant.values.byName(arg)));
	return CanvasKitVariantFilter(allowList: variants);
	}

	SuiteNameFilter? makeSuiteNameFilter() {
	final List<String>? suiteNameArgs = argResults!['suite'] as List<String>?;
	if (suiteNameArgs == null \|\| suiteNameArgs.isEmpty) {
	return null;
	}

	final Iterable<String> allSuiteNames = config.testSuites.map((TestSuite suite) => suite.name);
	for (final String suiteName in suiteNameArgs) {
	if (!allSuiteNames.contains(suiteName)) {
	throw ToolExit('No suite found named $suiteName');
	}
	}
	return SuiteNameFilter(allowList: Set<String>.from(suiteNameArgs));
	}

	BundleNameFilter? makeBundleNameFilter() {
	final List<String>? bundleNameArgs = argResults!['bundle'] as List<String>?;
	if (bundleNameArgs == null \|\| bundleNameArgs.isEmpty) {
	return null;
	}

	final Iterable<String> allBundleNames = config.testSuites.map(
	(TestSuite suite) => suite.testBundle.name
	);
	for (final String bundleName in bundleNameArgs) {
	if (!allBundleNames.contains(bundleName)) {
	throw ToolExit('No bundle found named $bundleName');
	}
	}
	return BundleNameFilter(allowList: Set<String>.from(bundleNameArgs));
	}

	FileFilter? makeFileFilter() {
	final List<FilePath> tests = targetFiles;
	if (tests.isEmpty) {
	return null;
	}
	final Set<String> bundleNames = <String>{};
	for (final FilePath testPath in tests) {
	if (!io.File(testPath.absolute).existsSync()) {
	throw ToolExit('Test path not found: $testPath');
	}
	bool bundleFound = false;
	for (final TestBundle bundle in config.testBundles) {
	final String testSetPath = getTestSetDirectory(bundle.testSet).path;
	if (path.isWithin(testSetPath, testPath.absolute)) {
	bundleFound = true;
	bundleNames.add(bundle.name);
	}
	}
	if (!bundleFound) {
	throw ToolExit('Test path not in any known test bundle: $testPath');
	}
	}
	return FileFilter(allowList: bundleNames);
	}

	List<SuiteFilter> get suiteFilters {
	final BrowserSuiteFilter? browserFilter = makeBrowserFilter();
	final CompilerFilter? compilerFilter = makeCompilerFilter();
	final RendererFilter? rendererFilter = makeRendererFilter();
	final CanvasKitVariantFilter? canvaskitVariantFilter = makeCanvasKitVariantFilter();
	final SuiteNameFilter? suiteNameFilter = makeSuiteNameFilter();
	final BundleNameFilter? bundleNameFilter = makeBundleNameFilter();
	final FileFilter? fileFilter = makeFileFilter();
	return <SuiteFilter>[
	PlatformBrowserFilter(),
	if (browserFilter != null) browserFilter,
	if (compilerFilter != null) compilerFilter,
	if (rendererFilter != null) rendererFilter,
	if (canvaskitVariantFilter != null) canvaskitVariantFilter,
	if (suiteNameFilter != null) suiteNameFilter,
	if (bundleNameFilter != null) bundleNameFilter,
	if (fileFilter != null) fileFilter,
	];
	}

	List<TestSuite> _filterTestSuites() {
	if (isVerbose) {
	print('Filtering suites...');
	}
	final List<SuiteFilter> filters = suiteFilters;
	final List<TestSuite> filteredSuites = config.testSuites.where((TestSuite suite) {
	for (final SuiteFilter filter in filters) {
	final SuiteFilterResult result = filter.filterSuite(suite);
	if (!result.isAccepted) {
	if (isVerbose) {
	print(' ${suite.name.ansiCyan} rejected for reason: ${result.rejectReason}');
	}
	return false;
	}
	}
	return true;
	}).toList();
	return filteredSuites;
	}

	List<TestBundle> _filterBundlesForSuites(List<TestSuite> suites) {
	final Set<TestBundle> seenBundles =
	Set<TestBundle>.from(suites.map((TestSuite suite) => suite.testBundle));
	return config.testBundles.where((TestBundle bundle) => seenBundles.contains(bundle)).toList();
	}

	ArtifactDependencies _artifactsForSuites(List<TestSuite> suites) {
	return suites.fold(ArtifactDependencies.none(),
	(ArtifactDependencies deps, TestSuite suite) => deps \| suite.artifactDependencies);
	}

	@override
	Future<bool> run() async {
	final List<TestSuite> filteredSuites = _filterTestSuites();
	final List<TestBundle> bundles = _filterBundlesForSuites(filteredSuites);
	final ArtifactDependencies artifacts = _artifactsForSuites(filteredSuites);
	if (boolArg('generate-builder-json')) {
	final String configString = generateBuilderJson(config);
	final io.File configFile = io.File(path.join(
	environment.flutterDirectory.path,
	'ci',
	'builders',
	'linux_web_engine.json',
	));
	configFile.writeAsStringSync(configString);
	return true;
	}
	if (isList \|\| isVerbose) {
	print('Suites:');
	for (final TestSuite suite in filteredSuites) {
	print(' ${suite.name.ansiCyan}');
	}
	print('Bundles:');
	for (final TestBundle bundle in bundles) {
	print(' ${bundle.name.ansiMagenta}');
	}
	print('Artifacts:');
	if (artifacts.canvasKit) {
	print(' canvaskit'.ansiYellow);
	}
	if (artifacts.canvasKitChromium) {
	print(' canvaskit_chromium'.ansiYellow);
	}
	if (artifacts.skwasm) {
	print(' skwasm'.ansiYellow);
	}
	}
	if (isList) {
	return true;
	}

	bool shouldRun = boolArg('run');
	bool shouldCompile = boolArg('compile');
	bool shouldCopyArtifacts = boolArg('copy-artifacts');
	if (!shouldRun && !shouldCompile && !shouldCopyArtifacts) {
	// If none of these is specified, we should assume we need to do all of them.
	shouldRun = true;
	shouldCompile = true;
	shouldCopyArtifacts = true;
	}

	final Set<FilePath>? testFiles = targetFiles.isEmpty ? null : Set<FilePath>.from(targetFiles);
	final Pipeline testPipeline = Pipeline(steps: <PipelineStep>[
	if (isWatchMode) ClearTerminalScreenStep(),
	if (shouldCopyArtifacts) CopyArtifactsStep(artifacts, runtimeMode: runtimeMode),
	if (shouldCompile)
	for (final TestBundle bundle in bundles)
	CompileBundleStep(
	bundle: bundle,
	isVerbose: isVerbose,
	testFiles: testFiles,
	),
	if (shouldRun)
	for (final TestSuite suite in filteredSuites)
	RunSuiteStep(
	suite,
	startPaused: startPaused,
	isVerbose: isVerbose,
	doUpdateScreenshotGoldens: doUpdateScreenshotGoldens,
	requireSkiaGold: requireSkiaGold,
	overridePathToCanvasKit: overridePathToCanvasKit,
	testFiles: testFiles,
	useDwarf: boolArg('dwarf'),
	),
	]);

	try {
	await testPipeline.run();
	if (isWatchMode) {
	print('');
	print('Initial test succeeded!');
	}
	} catch(error, stackTrace) {
	if (isWatchMode) {
	// The error is printed but not rethrown in watch mode because
	// failures are expected. The idea is that the developer corrects the
	// error, saves the file, and the pipeline reruns.
	print('');
	print('Initial test failed!\n');
	print(error);
	print(stackTrace);
	} else {
	rethrow;
	}
	}

	if (isWatchMode) {
	final FilePath dir = FilePath.fromWebUi('');
	print('');
	print(
	'Watching ${dir.relativeToCwd}/lib and ${dir.relativeToCwd}/test to re-run tests');
	print('');
	await PipelineWatcher(
	dir: dir.absolute,
	pipeline: testPipeline,
	ignore: (WatchEvent event) {
	// Ignore font files that are copied whenever tests run.
	if (event.path.endsWith('.ttf')) {
	return true;
	}

	// React to changes in lib/ and test/ folders.
	final String relativePath =
	path.relative(event.path, from: dir.absolute);
	if (path.isWithin('lib', relativePath) \|\|
	path.isWithin('test', relativePath)) {
	return false;
	}

	// Ignore anything else.
	return true;
	}).start();
	}
	return true;
	}
	}

	/// Clears the terminal screen and places the cursor at the top left corner.
	///
	/// This works on Linux and Mac. On Windows, it's a no-op.
	class ClearTerminalScreenStep implements PipelineStep {
	@override
	String get description => 'clearing terminal screen';

	@override
	bool get isSafeToInterrupt => false;

	@override
	Future<void> interrupt() async {}

	@override
	Future<void> run() async {
	if (!io.Platform.isWindows) {
	// See: https://en.wikipedia.org/wiki/ANSI_escape_code#CSI_sequences
	print('\x1B[2J\x1B[1;2H');
	}
	}
	}