lib/web_ui/dev/steps/run_suite_step.dart - mirrors/engine - Git at Google

 // Copyright 2013 The Flutter Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 import 'dart:convert';
 import 'dart:io' as io;

 import 'package:path/path.dart' as pathlib;
 // TODO(yjbanov): remove hacks when this is fixed:
 //                https://github.com/dart-lang/test/issues/1521
 import 'package:skia_gold_client/skia_gold_client.dart';
 import 'package:test_api/backend.dart' as hack;
 // TODO(ditman): Fix ignores when https://github.com/flutter/flutter/issues/143599 is resolved.
 import 'package:test_core/src/executable.dart' as test; // ignore: implementation_imports
 import 'package:test_core/src/runner/hack_register_platform.dart' as hack; // ignore: implementation_imports

 import '../browser.dart';
 import '../common.dart';
 import '../environment.dart';
 import '../exceptions.dart';
 import '../felt_config.dart';
 import '../pipeline.dart';
 import '../test_platform.dart';
 import '../utils.dart';

 /// Runs a test suite.
 ///
 /// Assumes the artifacts from previous steps are available, either from
 /// running them prior to this step locally, or by having the build graph copy
 /// them from another bot.
 class RunSuiteStep implements PipelineStep {
   RunSuiteStep(this.suite, {
     required this.startPaused,
     required this.isVerbose,
     required this.doUpdateScreenshotGoldens,
     required this.requireSkiaGold,
     required this.overridePathToCanvasKit,
     required this.useDwarf,
     this.testFiles,
   });

   final TestSuite suite;
   final Set<FilePath>? testFiles;
   final bool startPaused;
   final bool isVerbose;
   final bool doUpdateScreenshotGoldens;
   final String? overridePathToCanvasKit;
   final bool useDwarf;

   /// Require Skia Gold to be available and reachable.
   final bool requireSkiaGold;

   @override
   String get description => 'run_suite';

   @override
   bool get isSafeToInterrupt => true;

   @override
   Future<void> interrupt() async {}

   @override
   Future<void> run() async {
     _prepareTestResultsDirectory();
     final BrowserEnvironment browserEnvironment = getBrowserEnvironment(
       suite.runConfig.browser,
       useDwarf: useDwarf,
     );
     await browserEnvironment.prepare();

     final SkiaGoldClient? skiaClient = await _createSkiaClient();
     final String configurationFilePath = pathlib.join(
       environment.webUiRootDir.path,
       browserEnvironment.packageTestConfigurationYamlFile,
     );
     final String bundleBuildPath = getBundleBuildDirectory(suite.testBundle).path;
     final List<String> testArgs = <String>[
       ...<String>['-r', 'compact'],
       // Disable concurrency. Running with concurrency proved to be flaky.
       '--concurrency=1',
       if (startPaused) '--pause-after-load',
       '--platform=${browserEnvironment.packageTestRuntime.identifier}',
       '--precompiled=$bundleBuildPath',
       '--configuration=$configurationFilePath',
       if (AnsiColors.shouldEscape) '--color' else '--no-color',

       // TODO(jacksongardner): Set the default timeout to five minutes when
       // https://github.com/dart-lang/test/issues/2006 is fixed.
       '--',
       ..._collectTestPaths(),
     ];

     hack.registerPlatformPlugin(<hack.Runtime>[
       browserEnvironment.packageTestRuntime,
     ], () {
       return BrowserPlatform.start(
         suite,
         browserEnvironment: browserEnvironment,
         doUpdateScreenshotGoldens: doUpdateScreenshotGoldens,
         skiaClient: skiaClient,
         overridePathToCanvasKit: overridePathToCanvasKit,
         isVerbose: isVerbose,
       );
     });

     print('[${suite.name.ansiCyan}] Running...');

     // We want to run tests with the test set's directory as a working directory.
     final io.Directory testSetDirectory = io.Directory(pathlib.join(
       environment.webUiTestDir.path,
       suite.testBundle.testSet.directory,
     ));
     final dynamic originalCwd = io.Directory.current;
     io.Directory.current = testSetDirectory;
     try {
       await test.main(testArgs);
     } finally {
       io.Directory.current = originalCwd;
     }

     await browserEnvironment.cleanup();

     // Since we are just calling `main()` on the test executable, it will modify
     // the exit code. We use this as a signal that there were some tests that failed.
     if (io.exitCode != 0) {
       print('[${suite.name.ansiCyan}] ${'Some tests failed.'.ansiRed}');
       // Change the exit code back to 0 when we're done. Failures will be bubbled up
       // at the end of the pipeline and we'll exit abnormally if there were any
       // failures in the pipeline.
       io.exitCode = 0;
       throw ToolExit('Some unit tests failed in suite ${suite.name.ansiCyan}.');
     } else {
       print('[${suite.name.ansiCyan}] ${'All tests passed!'.ansiGreen}');
     }
   }

   io.Directory _prepareTestResultsDirectory() {
     final io.Directory resultsDirectory = io.Directory(pathlib.join(
       environment.webUiTestResultsDirectory.path,
       suite.name,
     ));
     if (resultsDirectory.existsSync()) {
       resultsDirectory.deleteSync(recursive: true);
     }
     resultsDirectory.createSync(recursive: true);
     return resultsDirectory;
   }

   List<String> _collectTestPaths() {
     final io.Directory bundleBuild = getBundleBuildDirectory(suite.testBundle);
     final io.File resultsJsonFile = io.File(pathlib.join(
       bundleBuild.path,
       'results.json',
     ));
     if (!resultsJsonFile.existsSync()) {
       throw ToolExit('Could not find built bundle ${suite.testBundle.name.ansiMagenta} for suite ${suite.name.ansiCyan}.');
     }
     final String jsonString = resultsJsonFile.readAsStringSync();
     final dynamic jsonContents = const JsonDecoder().convert(jsonString);
     final dynamic results = jsonContents['results'];
     final List<String> testPaths = <String>[];
     results.forEach((dynamic k, dynamic v) {
       final String result = v as String;
       final String testPath = k as String;
       if (testFiles != null) {
         if (!testFiles!.contains(FilePath.fromTestSet(suite.testBundle.testSet, testPath))) {
           return;
         }
       }
       if (result == 'success') {
         testPaths.add(testPath);
       }
     });
     return testPaths;
   }

   Future<SkiaGoldClient?> _createSkiaClient() async {
     if (suite.testBundle.compileConfigs.length > 1) {
       // Multiple compile configs are only used for our fallback tests, which
       // do not collect goldens.
       return null;
     }
     if (suite.runConfig.browser == BrowserName.safari) {
       // Goldens from Safari produce too many diffs, disabled for now.
       // See https://github.com/flutter/flutter/issues/143591
       return null;
     }
     final Renderer renderer = suite.testBundle.compileConfigs.first.renderer;
     final CanvasKitVariant? variant = suite.runConfig.variant;
     final io.Directory workDirectory = getSkiaGoldDirectoryForSuite(suite);
     if (workDirectory.existsSync()) {
       workDirectory.deleteSync(recursive: true);
     }
     final bool isWasm = suite.testBundle.compileConfigs.first.compiler == Compiler.dart2wasm;
     final SkiaGoldClient skiaClient = SkiaGoldClient(
       workDirectory,
       dimensions: <String, String> {
         'Browser': suite.runConfig.browser.name,
         if (isWasm) 'Wasm': 'true',
         'Renderer': renderer.name,
         if (variant != null) 'CanvasKitVariant': variant.name,
       },
     );

     if (await _checkSkiaClient(skiaClient)) {
       return skiaClient;
     }

     if (requireSkiaGold) {
       throw ToolExit('Skia Gold is required but is unavailable.');
     }

     return null;
   }

   /// Checks whether the Skia Client is usable in this environment.
   Future<bool> _checkSkiaClient(SkiaGoldClient skiaClient) async {
     // Now let's check whether Skia Gold is reachable or not.
     if (isLuci) {
       if (isSkiaGoldClientAvailable) {
         try {
           await skiaClient.auth();
           return true;
         } catch (e) {
           print(e);
         }
       }
     } else {
       try {
         // Check if we can reach Gold.
         await skiaClient.getExpectationForTest('');
         return true;
       } on io.OSError catch (_) {
         print('OSError occurred, could not reach Gold.');
       } on io.SocketException catch (_) {
         print('SocketException occurred, could not reach Gold.');
       }
     }

     return false;
   }
 }
	// Copyright 2013 The Flutter Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	import 'dart:convert';
	import 'dart:io' as io;

	import 'package:path/path.dart' as pathlib;
	// TODO(yjbanov): remove hacks when this is fixed:
	// https://github.com/dart-lang/test/issues/1521
	import 'package:skia_gold_client/skia_gold_client.dart';
	import 'package:test_api/backend.dart' as hack;
	// TODO(ditman): Fix ignores when https://github.com/flutter/flutter/issues/143599 is resolved.
	import 'package:test_core/src/executable.dart' as test; // ignore: implementation_imports
	import 'package:test_core/src/runner/hack_register_platform.dart' as hack; // ignore: implementation_imports

	import '../browser.dart';
	import '../common.dart';
	import '../environment.dart';
	import '../exceptions.dart';
	import '../felt_config.dart';
	import '../pipeline.dart';
	import '../test_platform.dart';
	import '../utils.dart';

	/// Runs a test suite.
	///
	/// Assumes the artifacts from previous steps are available, either from
	/// running them prior to this step locally, or by having the build graph copy
	/// them from another bot.
	class RunSuiteStep implements PipelineStep {
	RunSuiteStep(this.suite, {
	required this.startPaused,
	required this.isVerbose,
	required this.doUpdateScreenshotGoldens,
	required this.requireSkiaGold,
	required this.overridePathToCanvasKit,
	required this.useDwarf,
	this.testFiles,
	});

	final TestSuite suite;
	final Set<FilePath>? testFiles;
	final bool startPaused;
	final bool isVerbose;
	final bool doUpdateScreenshotGoldens;
	final String? overridePathToCanvasKit;
	final bool useDwarf;

	/// Require Skia Gold to be available and reachable.
	final bool requireSkiaGold;

	@override
	String get description => 'run_suite';

	@override
	bool get isSafeToInterrupt => true;

	@override
	Future<void> interrupt() async {}

	@override
	Future<void> run() async {
	_prepareTestResultsDirectory();
	final BrowserEnvironment browserEnvironment = getBrowserEnvironment(
	suite.runConfig.browser,
	useDwarf: useDwarf,
	);
	await browserEnvironment.prepare();

	final SkiaGoldClient? skiaClient = await _createSkiaClient();
	final String configurationFilePath = pathlib.join(
	environment.webUiRootDir.path,
	browserEnvironment.packageTestConfigurationYamlFile,
	);
	final String bundleBuildPath = getBundleBuildDirectory(suite.testBundle).path;
	final List<String> testArgs = <String>[
	...<String>['-r', 'compact'],
	// Disable concurrency. Running with concurrency proved to be flaky.
	'--concurrency=1',
	if (startPaused) '--pause-after-load',
	'--platform=${browserEnvironment.packageTestRuntime.identifier}',
	'--precompiled=$bundleBuildPath',
	'--configuration=$configurationFilePath',
	if (AnsiColors.shouldEscape) '--color' else '--no-color',

	// TODO(jacksongardner): Set the default timeout to five minutes when
	// https://github.com/dart-lang/test/issues/2006 is fixed.
	'--',
	..._collectTestPaths(),
	];

	hack.registerPlatformPlugin(<hack.Runtime>[
	browserEnvironment.packageTestRuntime,
	], () {
	return BrowserPlatform.start(
	suite,
	browserEnvironment: browserEnvironment,
	doUpdateScreenshotGoldens: doUpdateScreenshotGoldens,
	skiaClient: skiaClient,
	overridePathToCanvasKit: overridePathToCanvasKit,
	isVerbose: isVerbose,
	);
	});

	print('[${suite.name.ansiCyan}] Running...');

	// We want to run tests with the test set's directory as a working directory.
	final io.Directory testSetDirectory = io.Directory(pathlib.join(
	environment.webUiTestDir.path,
	suite.testBundle.testSet.directory,
	));
	final dynamic originalCwd = io.Directory.current;
	io.Directory.current = testSetDirectory;
	try {
	await test.main(testArgs);
	} finally {
	io.Directory.current = originalCwd;
	}

	await browserEnvironment.cleanup();

	// Since we are just calling `main()` on the test executable, it will modify
	// the exit code. We use this as a signal that there were some tests that failed.
	if (io.exitCode != 0) {
	print('[${suite.name.ansiCyan}] ${'Some tests failed.'.ansiRed}');
	// Change the exit code back to 0 when we're done. Failures will be bubbled up
	// at the end of the pipeline and we'll exit abnormally if there were any
	// failures in the pipeline.
	io.exitCode = 0;
	throw ToolExit('Some unit tests failed in suite ${suite.name.ansiCyan}.');
	} else {
	print('[${suite.name.ansiCyan}] ${'All tests passed!'.ansiGreen}');
	}
	}

	io.Directory _prepareTestResultsDirectory() {
	final io.Directory resultsDirectory = io.Directory(pathlib.join(
	environment.webUiTestResultsDirectory.path,
	suite.name,
	));
	if (resultsDirectory.existsSync()) {
	resultsDirectory.deleteSync(recursive: true);
	}
	resultsDirectory.createSync(recursive: true);
	return resultsDirectory;
	}

	List<String> _collectTestPaths() {
	final io.Directory bundleBuild = getBundleBuildDirectory(suite.testBundle);
	final io.File resultsJsonFile = io.File(pathlib.join(
	bundleBuild.path,
	'results.json',
	));
	if (!resultsJsonFile.existsSync()) {
	throw ToolExit('Could not find built bundle ${suite.testBundle.name.ansiMagenta} for suite ${suite.name.ansiCyan}.');
	}
	final String jsonString = resultsJsonFile.readAsStringSync();
	final dynamic jsonContents = const JsonDecoder().convert(jsonString);
	final dynamic results = jsonContents['results'];
	final List<String> testPaths = <String>[];
	results.forEach((dynamic k, dynamic v) {
	final String result = v as String;
	final String testPath = k as String;
	if (testFiles != null) {
	if (!testFiles!.contains(FilePath.fromTestSet(suite.testBundle.testSet, testPath))) {
	return;
	}
	}
	if (result == 'success') {
	testPaths.add(testPath);
	}
	});
	return testPaths;
	}

	Future<SkiaGoldClient?> _createSkiaClient() async {
	if (suite.testBundle.compileConfigs.length > 1) {
	// Multiple compile configs are only used for our fallback tests, which
	// do not collect goldens.
	return null;
	}
	if (suite.runConfig.browser == BrowserName.safari) {
	// Goldens from Safari produce too many diffs, disabled for now.
	// See https://github.com/flutter/flutter/issues/143591
	return null;
	}
	final Renderer renderer = suite.testBundle.compileConfigs.first.renderer;
	final CanvasKitVariant? variant = suite.runConfig.variant;
	final io.Directory workDirectory = getSkiaGoldDirectoryForSuite(suite);
	if (workDirectory.existsSync()) {
	workDirectory.deleteSync(recursive: true);
	}
	final bool isWasm = suite.testBundle.compileConfigs.first.compiler == Compiler.dart2wasm;
	final SkiaGoldClient skiaClient = SkiaGoldClient(
	workDirectory,
	dimensions: <String, String> {
	'Browser': suite.runConfig.browser.name,
	if (isWasm) 'Wasm': 'true',
	'Renderer': renderer.name,
	if (variant != null) 'CanvasKitVariant': variant.name,
	},
	);

	if (await _checkSkiaClient(skiaClient)) {
	return skiaClient;
	}

	if (requireSkiaGold) {
	throw ToolExit('Skia Gold is required but is unavailable.');
	}

	return null;
	}

	/// Checks whether the Skia Client is usable in this environment.
	Future<bool> _checkSkiaClient(SkiaGoldClient skiaClient) async {
	// Now let's check whether Skia Gold is reachable or not.
	if (isLuci) {
	if (isSkiaGoldClientAvailable) {
	try {
	await skiaClient.auth();
	return true;
	} catch (e) {
	print(e);
	}
	}
	} else {
	try {
	// Check if we can reach Gold.
	await skiaClient.getExpectationForTest('');
	return true;
	} on io.OSError catch (_) {
	print('OSError occurred, could not reach Gold.');
	} on io.SocketException catch (_) {
	print('SocketException occurred, could not reach Gold.');
	}
	}

	return false;
	}
	}