| // Copyright 2014 The Flutter Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| import 'dart:async' show FutureOr; |
| import 'dart:io' as io show OSError, SocketException; |
| import 'dart:math' as math show Random; |
| import 'dart:typed_data' show Uint8List; |
| |
| import 'package:file/file.dart'; |
| import 'package:file/local.dart'; |
| import 'package:flutter/foundation.dart'; |
| import 'package:flutter_goldens_client/skia_client.dart'; |
| import 'package:flutter_test/flutter_test.dart'; |
| import 'package:platform/platform.dart'; |
| |
| export 'package:flutter_goldens_client/skia_client.dart'; |
| |
| // If you are here trying to figure out how to use golden files in the Flutter |
| // repo itself, consider reading this wiki page: |
| // https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package%3Aflutter |
| |
| const String _kFlutterRootKey = 'FLUTTER_ROOT'; |
| |
| /// Main method that can be used in a `flutter_test_config.dart` file to set |
| /// [goldenFileComparator] to an instance of [FlutterGoldenFileComparator] that |
| /// works for the current test. _Which_ FlutterGoldenFileComparator is |
| /// instantiated is based on the current testing environment. |
| Future<void> testExecutable(FutureOr<void> Function() testMain) async { |
| const Platform platform = LocalPlatform(); |
| if (FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform)) { |
| goldenFileComparator = await FlutterPostSubmitFileComparator.fromDefaultComparator(platform); |
| } else if (FlutterPreSubmitFileComparator.isAvailableForEnvironment(platform)) { |
| goldenFileComparator = await FlutterPreSubmitFileComparator.fromDefaultComparator(platform); |
| } else if (FlutterSkippingFileComparator.isAvailableForEnvironment(platform)) { |
| goldenFileComparator = FlutterSkippingFileComparator.fromDefaultComparator( |
| 'Golden file testing is not executed on Cirrus, or LUCI environments outside of flutter/flutter.' |
| ); |
| } else { |
| goldenFileComparator = await FlutterLocalFileComparator.fromDefaultComparator(platform); |
| } |
| |
| await testMain(); |
| } |
| |
| /// Abstract base class golden file comparator specific to the `flutter/flutter` |
| /// repository. |
| /// |
| /// Golden file testing for the `flutter/flutter` repository is handled by three |
| /// different [FlutterGoldenFileComparator]s, depending on the current testing |
| /// environment. |
| /// |
| /// * The [FlutterPostSubmitFileComparator] is utilized during post-submit |
| /// testing, after a pull request has landed on the master branch. This |
| /// comparator uses the [SkiaGoldClient] and the `goldctl` tool to upload |
| /// tests to the [Flutter Gold dashboard](https://flutter-gold.skia.org). |
| /// Flutter Gold manages the master golden files for the `flutter/flutter` |
| /// repository. |
| /// |
| /// * The [FlutterPreSubmitFileComparator] is utilized in pre-submit testing, |
| /// before a pull request lands on the master branch. This |
| /// comparator uses the [SkiaGoldClient] to execute tryjobs, allowing |
| /// contributors to view and check in visual differences before landing the |
| /// change. |
| /// |
| /// * The [FlutterLocalFileComparator] is used for local development testing. |
| /// This comparator will use the [SkiaGoldClient] to request baseline images |
| /// from [Flutter Gold](https://flutter-gold.skia.org) and manually compare |
| /// pixels. If a difference is detected, this comparator will |
| /// generate failure output illustrating the found difference. If a baseline |
| /// is not found for a given test image, it will consider it a new test and |
| /// output the new image for verification. |
| /// |
| /// The [FlutterSkippingFileComparator] is utilized to skip tests outside |
| /// of the appropriate environments described above. Currently, some Luci |
| /// environments do not execute golden file testing, and as such do not require |
| /// a comparator. This comparator is also used when an internet connection is |
| /// unavailable. |
| abstract class FlutterGoldenFileComparator extends GoldenFileComparator { |
| /// Creates a [FlutterGoldenFileComparator] that will resolve golden file |
| /// URIs relative to the specified [basedir], and retrieve golden baselines |
| /// using the [skiaClient]. The [basedir] is used for writing and accessing |
| /// information and files for interacting with the [skiaClient]. When testing |
| /// locally, the [basedir] will also contain any diffs from failed tests, or |
| /// goldens generated from newly introduced tests. |
| /// |
| /// The [fs] and [platform] parameters are useful in tests, where the default |
| /// file system and platform can be replaced by mock instances. |
| @visibleForTesting |
| FlutterGoldenFileComparator( |
| this.basedir, |
| this.skiaClient, { |
| this.fs = const LocalFileSystem(), |
| this.platform = const LocalPlatform(), |
| }); |
| |
| /// The directory to which golden file URIs will be resolved in [compare] and |
| /// [update], cannot be null. |
| final Uri basedir; |
| |
| /// A client for uploading image tests and making baseline requests to the |
| /// Flutter Gold Dashboard, cannot be null. |
| final SkiaGoldClient skiaClient; |
| |
| /// The file system used to perform file access. |
| @visibleForTesting |
| final FileSystem fs; |
| |
| /// A wrapper for the [dart:io.Platform] API. |
| @visibleForTesting |
| final Platform platform; |
| |
| @override |
| Future<void> update(Uri golden, Uint8List imageBytes) async { |
| final File goldenFile = getGoldenFile(golden); |
| await goldenFile.parent.create(recursive: true); |
| await goldenFile.writeAsBytes(imageBytes, flush: true); |
| } |
| |
| @override |
| Uri getTestUri(Uri key, int? version) => key; |
| |
| /// Calculate the appropriate basedir for the current test context. |
| /// |
| /// The optional [suffix] argument is used by the |
| /// [FlutterPostSubmitFileComparator] and the [FlutterPreSubmitFileComparator]. |
| /// These [FlutterGoldenFileComparators] randomize their base directories to |
| /// maintain thread safety while using the `goldctl` tool. |
| @protected |
| @visibleForTesting |
| static Directory getBaseDirectory( |
| LocalFileComparator defaultComparator, |
| Platform platform, { |
| String suffix = '', |
| bool local = false, |
| }) { |
| const FileSystem fs = LocalFileSystem(); |
| final Directory flutterRoot = fs.directory(platform.environment[_kFlutterRootKey]); |
| Directory comparisonRoot; |
| |
| if (!local) { |
| comparisonRoot = fs.systemTempDirectory.childDirectory( |
| 'skia_goldens_$suffix' |
| ); |
| } else { |
| comparisonRoot = flutterRoot.childDirectory( |
| fs.path.join( |
| 'bin', |
| 'cache', |
| 'pkg', |
| 'skia_goldens', |
| ) |
| ); |
| } |
| |
| final Directory testDirectory = fs.directory(defaultComparator.basedir); |
| final String testDirectoryRelativePath = fs.path.relative( |
| testDirectory.path, |
| from: flutterRoot.path, |
| ); |
| return comparisonRoot.childDirectory(testDirectoryRelativePath); |
| } |
| |
| /// Returns the golden [File] identified by the given [Uri]. |
| @protected |
| File getGoldenFile(Uri uri) { |
| final File goldenFile = fs.directory(basedir).childFile(fs.file(uri).path); |
| return goldenFile; |
| } |
| |
| /// Prepends the golden URL with the library name that encloses the current |
| /// test. |
| Uri _addPrefix(Uri golden) { |
| final String prefix = basedir.pathSegments[basedir.pathSegments.length - 2]; |
| return Uri.parse(prefix + '.' + golden.toString()); |
| } |
| } |
| |
| /// A [FlutterGoldenFileComparator] for testing golden images with Skia Gold in |
| /// post-submit. |
| /// |
| /// For testing across all platforms, the [SkiaGoldClient] is used to upload |
| /// images for framework-related golden tests and process results. |
| /// |
| /// See also: |
| /// |
| /// * [GoldenFileComparator], the abstract class that |
| /// [FlutterGoldenFileComparator] implements. |
| /// * [FlutterPreSubmitFileComparator], another |
| /// [FlutterGoldenFileComparator] that tests golden images before changes are |
| /// merged into the master branch. |
| /// * [FlutterLocalFileComparator], another |
| /// [FlutterGoldenFileComparator] that tests golden images locally on your |
| /// current machine. |
| class FlutterPostSubmitFileComparator extends FlutterGoldenFileComparator { |
| /// Creates a [FlutterPostSubmitFileComparator] that will test golden file |
| /// images against Skia Gold. |
| /// |
| /// The [fs] and [platform] parameters are useful in tests, where the default |
| /// file system and platform can be replaced by mock instances. |
| FlutterPostSubmitFileComparator( |
| final Uri basedir, |
| final SkiaGoldClient skiaClient, { |
| final FileSystem fs = const LocalFileSystem(), |
| final Platform platform = const LocalPlatform(), |
| }) : super( |
| basedir, |
| skiaClient, |
| fs: fs, |
| platform: platform, |
| ); |
| |
| /// Creates a new [FlutterPostSubmitFileComparator] that mirrors the relative |
| /// path resolution of the default [goldenFileComparator]. |
| /// |
| /// The [goldens] and [defaultComparator] parameters are visible for testing |
| /// purposes only. |
| static Future<FlutterPostSubmitFileComparator> fromDefaultComparator( |
| final Platform platform, { |
| SkiaGoldClient? goldens, |
| LocalFileComparator? defaultComparator, |
| }) async { |
| |
| defaultComparator ??= goldenFileComparator as LocalFileComparator; |
| final Directory baseDirectory = FlutterGoldenFileComparator.getBaseDirectory( |
| defaultComparator, |
| platform, |
| suffix: '${math.Random().nextInt(10000)}', |
| ); |
| baseDirectory.createSync(recursive: true); |
| |
| goldens ??= SkiaGoldClient(baseDirectory); |
| await goldens.auth(); |
| await goldens.imgtestInit(); |
| return FlutterPostSubmitFileComparator(baseDirectory.uri, goldens); |
| } |
| |
| @override |
| Future<bool> compare(Uint8List imageBytes, Uri golden) async { |
| golden = _addPrefix(golden); |
| await update(golden, imageBytes); |
| final File goldenFile = getGoldenFile(golden); |
| |
| return skiaClient.imgtestAdd(golden.path, goldenFile); |
| } |
| |
| /// Decides based on the current environment if goldens tests should be |
| /// executed through Skia Gold. |
| static bool isAvailableForEnvironment(Platform platform) { |
| final bool luciPostSubmit = platform.environment.containsKey('SWARMING_TASK_ID') |
| && platform.environment.containsKey('GOLDCTL') |
| // Luci tryjob environments contain this value to inform the [FlutterPreSubmitComparator]. |
| && !platform.environment.containsKey('GOLD_TRYJOB'); |
| |
| return luciPostSubmit; |
| } |
| } |
| |
| /// A [FlutterGoldenFileComparator] for testing golden images before changes are |
| /// merged into the master branch. The comparator executes tryjobs using the |
| /// [SkiaGoldClient]. |
| /// |
| /// See also: |
| /// |
| /// * [GoldenFileComparator], the abstract class that |
| /// [FlutterGoldenFileComparator] implements. |
| /// * [FlutterPostSubmitFileComparator], another |
| /// [FlutterGoldenFileComparator] that uploads tests to the Skia Gold |
| /// dashboard in post-submit. |
| /// * [FlutterLocalFileComparator], another |
| /// [FlutterGoldenFileComparator] that tests golden images locally on your |
| /// current machine. |
| class FlutterPreSubmitFileComparator extends FlutterGoldenFileComparator { |
| /// Creates a [FlutterPreSubmitFileComparator] that will test golden file |
| /// images against baselines requested from Flutter Gold. |
| /// |
| /// The [fs] and [platform] parameters are useful in tests, where the default |
| /// file system and platform can be replaced by mock instances. |
| FlutterPreSubmitFileComparator( |
| final Uri basedir, |
| final SkiaGoldClient skiaClient, { |
| final FileSystem fs = const LocalFileSystem(), |
| final Platform platform = const LocalPlatform(), |
| }) : super( |
| basedir, |
| skiaClient, |
| fs: fs, |
| platform: platform, |
| ); |
| |
| /// Creates a new [FlutterPreSubmitFileComparator] that mirrors the |
| /// relative path resolution of the default [goldenFileComparator]. |
| /// |
| /// The [goldens] and [defaultComparator] parameters are visible for testing |
| /// purposes only. |
| static Future<FlutterGoldenFileComparator> fromDefaultComparator( |
| final Platform platform, { |
| SkiaGoldClient? goldens, |
| LocalFileComparator? defaultComparator, |
| Directory? testBasedir, |
| }) async { |
| |
| defaultComparator ??= goldenFileComparator as LocalFileComparator; |
| final Directory baseDirectory = testBasedir ?? FlutterGoldenFileComparator.getBaseDirectory( |
| defaultComparator, |
| platform, |
| suffix: '${math.Random().nextInt(10000)}', |
| ); |
| |
| if (!baseDirectory.existsSync()) |
| baseDirectory.createSync(recursive: true); |
| |
| goldens ??= SkiaGoldClient(baseDirectory); |
| |
| await goldens.auth(); |
| await goldens.tryjobInit(); |
| return FlutterPreSubmitFileComparator( |
| baseDirectory.uri, |
| goldens, |
| platform: platform, |
| ); |
| } |
| |
| @override |
| Future<bool> compare(Uint8List imageBytes, Uri golden) async { |
| golden = _addPrefix(golden); |
| await update(golden, imageBytes); |
| final File goldenFile = getGoldenFile(golden); |
| |
| await skiaClient.tryjobAdd(golden.path, goldenFile); |
| |
| // This will always return true since golden file test failures are managed |
| // in pre-submit checks by the flutter-gold status check. |
| return true; |
| } |
| |
| /// Decides based on the current environment if goldens tests should be |
| /// executed as pre-submit tests with Skia Gold. |
| static bool isAvailableForEnvironment(Platform platform) { |
| final bool luciPreSubmit = platform.environment.containsKey('SWARMING_TASK_ID') |
| && platform.environment.containsKey('GOLDCTL') |
| && platform.environment.containsKey('GOLD_TRYJOB'); |
| return luciPreSubmit; |
| } |
| } |
| |
| /// A [FlutterGoldenFileComparator] for testing conditions that do not execute |
| /// golden file tests. |
| /// |
| /// Currently, this comparator is used on Cirrus, or in Luci environments when executing tests |
| /// outside of the flutter/flutter repository. |
| /// |
| /// See also: |
| /// |
| /// * [FlutterPostSubmitFileComparator], another [FlutterGoldenFileComparator] |
| /// that tests golden images through Skia Gold. |
| /// * [FlutterPreSubmitFileComparator], another |
| /// [FlutterGoldenFileComparator] that tests golden images before changes are |
| /// merged into the master branch. |
| /// * [FlutterLocalFileComparator], another |
| /// [FlutterGoldenFileComparator] that tests golden images locally on your |
| /// current machine. |
| class FlutterSkippingFileComparator extends FlutterGoldenFileComparator { |
| /// Creates a [FlutterSkippingFileComparator] that will skip tests that |
| /// are not in the right environment for golden file testing. |
| FlutterSkippingFileComparator( |
| final Uri basedir, |
| final SkiaGoldClient skiaClient, |
| this.reason, |
| ) : super(basedir, skiaClient); |
| |
| /// Describes the reason for using the [FlutterSkippingFileComparator]. |
| /// |
| /// Cannot be null. |
| final String reason; |
| |
| /// Creates a new [FlutterSkippingFileComparator] that mirrors the |
| /// relative path resolution of the default [goldenFileComparator]. |
| static FlutterSkippingFileComparator fromDefaultComparator( |
| String reason, { |
| LocalFileComparator? defaultComparator, |
| }) { |
| defaultComparator ??= goldenFileComparator as LocalFileComparator; |
| const FileSystem fs = LocalFileSystem(); |
| final Uri basedir = defaultComparator.basedir; |
| final SkiaGoldClient skiaClient = SkiaGoldClient(fs.directory(basedir)); |
| return FlutterSkippingFileComparator(basedir, skiaClient, reason); |
| } |
| |
| @override |
| Future<bool> compare(Uint8List imageBytes, Uri golden) async { |
| print( |
| 'Skipping "$golden" test : $reason' |
| ); |
| return true; |
| } |
| |
| @override |
| Future<void> update(Uri golden, Uint8List imageBytes) async {} |
| |
| /// Decides, based on the current environment, if this comparator should be |
| /// used. |
| /// |
| /// If we are in a CI environment, LUCI or Cirrus, but are not using the other |
| /// comparators, we skip. |
| static bool isAvailableForEnvironment(Platform platform) { |
| return platform.environment.containsKey('SWARMING_TASK_ID') |
| // Some builds are still being run on Cirrus, we should skip these. |
| || platform.environment.containsKey('CIRRUS_CI'); |
| } |
| } |
| |
| /// A [FlutterGoldenFileComparator] for testing golden images locally on your |
| /// current machine. |
| /// |
| /// This comparator utilizes the [SkiaGoldClient] to request baseline images for |
| /// the given device under test for comparison. This comparator is initialized |
| /// when conditions for all other [FlutterGoldenFileComparators] have not been |
| /// met, see the `isAvailableForEnvironment` method for each one listed below. |
| /// |
| /// The [FlutterLocalFileComparator] is intended to run on local machines and |
| /// serve as a smoke test during development. As such, it will not be able to |
| /// detect unintended changes on environments other than the currently executing |
| /// machine, until they are tested using the [FlutterPreSubmitFileComparator]. |
| /// |
| /// See also: |
| /// |
| /// * [GoldenFileComparator], the abstract class that |
| /// [FlutterGoldenFileComparator] implements. |
| /// * [FlutterPostSubmitFileComparator], another |
| /// [FlutterGoldenFileComparator] that uploads tests to the Skia Gold |
| /// dashboard. |
| /// * [FlutterPreSubmitFileComparator], another |
| /// [FlutterGoldenFileComparator] that tests golden images before changes are |
| /// merged into the master branch. |
| /// * [FlutterSkippingFileComparator], another |
| /// [FlutterGoldenFileComparator] that controls post-submit testing |
| /// conditions that do not execute golden file tests. |
| class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalComparisonOutput { |
| /// Creates a [FlutterLocalFileComparator] that will test golden file |
| /// images against baselines requested from Flutter Gold. |
| /// |
| /// The [fs] and [platform] parameters are useful in tests, where the default |
| /// file system and platform can be replaced by mock instances. |
| FlutterLocalFileComparator( |
| final Uri basedir, |
| final SkiaGoldClient skiaClient, { |
| final FileSystem fs = const LocalFileSystem(), |
| final Platform platform = const LocalPlatform(), |
| }) : super( |
| basedir, |
| skiaClient, |
| fs: fs, |
| platform: platform, |
| ); |
| |
| /// Creates a new [FlutterLocalFileComparator] that mirrors the |
| /// relative path resolution of the default [goldenFileComparator]. |
| /// |
| /// The [goldens], [defaultComparator], and [baseDirectory] parameters are |
| /// visible for testing purposes only. |
| static Future<FlutterGoldenFileComparator> fromDefaultComparator( |
| final Platform platform, { |
| SkiaGoldClient? goldens, |
| LocalFileComparator? defaultComparator, |
| Directory? baseDirectory, |
| }) async { |
| defaultComparator ??= goldenFileComparator as LocalFileComparator; |
| baseDirectory ??= FlutterGoldenFileComparator.getBaseDirectory( |
| defaultComparator, |
| platform, |
| local: true, |
| ); |
| |
| if(!baseDirectory.existsSync()) { |
| baseDirectory.createSync(recursive: true); |
| } |
| |
| goldens ??= SkiaGoldClient(baseDirectory); |
| try { |
| // Check if we can reach Gold. |
| await goldens.getExpectationForTest(''); |
| } on io.OSError catch (_) { |
| return FlutterSkippingFileComparator( |
| baseDirectory.uri, |
| goldens, |
| 'OSError occurred, could not reach Gold. ' |
| 'Switching to FlutterSkippingGoldenFileComparator.', |
| ); |
| } on io.SocketException catch (_) { |
| return FlutterSkippingFileComparator( |
| baseDirectory.uri, |
| goldens, |
| 'SocketException occurred, could not reach Gold. ' |
| 'Switching to FlutterSkippingGoldenFileComparator.', |
| ); |
| } |
| |
| return FlutterLocalFileComparator(baseDirectory.uri, goldens); |
| } |
| |
| @override |
| Future<bool> compare(Uint8List imageBytes, Uri golden) async { |
| golden = _addPrefix(golden); |
| final String testName = skiaClient.cleanTestName(golden.path); |
| late String? testExpectation; |
| testExpectation = await skiaClient.getExpectationForTest(testName); |
| |
| if (testExpectation == null || testExpectation.isEmpty) { |
| // There is no baseline for this test |
| print('No expectations provided by Skia Gold for test: $golden. ' |
| 'This may be a new test. If this is an unexpected result, check ' |
| 'https://flutter-gold.skia.org.\n' |
| 'Validate image output found at $basedir' |
| ); |
| update(golden, imageBytes); |
| return true; |
| } |
| |
| ComparisonResult result; |
| final List<int> goldenBytes = await skiaClient.getImageBytes(testExpectation); |
| |
| result = await GoldenFileComparator.compareLists( |
| imageBytes, |
| goldenBytes, |
| ); |
| |
| if (result.passed) |
| return true; |
| |
| final String error = await generateFailureOutput(result, golden, basedir); |
| throw FlutterError(error); |
| } |
| } |