Ian Hickson | 449f4a6 | 2019-11-27 15:04:02 -0800 | [diff] [blame] | 1 | // Copyright 2014 The Flutter Authors. All rights reserved. |
Todd Volkert | 65079ad | 2018-05-03 07:39:41 -0700 | [diff] [blame] | 2 | // Use of this source code is governed by a BSD-style license that can be |
| 3 | // found in the LICENSE file. |
Kate Lovett | 9e3e44e | 2019-04-19 11:32:16 -0700 | [diff] [blame] | 4 | |
Todd Volkert | 65079ad | 2018-05-03 07:39:41 -0700 | [diff] [blame] | 5 | import 'dart:async'; |
Kate Lovett | b1ca7f4 | 2019-11-15 13:04:59 -0800 | [diff] [blame] | 6 | import 'dart:io' as io; |
Kate Lovett | 1374a41 | 2019-11-27 11:29:53 -0800 | [diff] [blame] | 7 | import 'dart:math' as math; |
Todd Volkert | 65079ad | 2018-05-03 07:39:41 -0700 | [diff] [blame] | 8 | import 'dart:typed_data'; |
| 9 | |
Todd Volkert | 65079ad | 2018-05-03 07:39:41 -0700 | [diff] [blame] | 10 | import 'package:file/file.dart'; |
| 11 | import 'package:file/local.dart'; |
| 12 | import 'package:flutter_test/flutter_test.dart'; |
| 13 | import 'package:meta/meta.dart'; |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 14 | import 'package:platform/platform.dart'; |
Todd Volkert | 65079ad | 2018-05-03 07:39:41 -0700 | [diff] [blame] | 15 | |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 16 | import 'package:flutter_goldens_client/skia_client.dart'; |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 17 | export 'package:flutter_goldens_client/skia_client.dart'; |
Kate Lovett | 7e542fc | 2019-04-19 14:33:49 -0700 | [diff] [blame] | 18 | |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 19 | // If you are here trying to figure out how to use golden files in the Flutter |
| 20 | // repo itself, consider reading this wiki page: |
| 21 | // https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package%3Aflutter |
| 22 | |
| 23 | const String _kFlutterRootKey = 'FLUTTER_ROOT'; |
| 24 | |
Todd Volkert | 65079ad | 2018-05-03 07:39:41 -0700 | [diff] [blame] | 25 | /// Main method that can be used in a `flutter_test_config.dart` file to set |
| 26 | /// [goldenFileComparator] to an instance of [FlutterGoldenFileComparator] that |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 27 | /// works for the current test. _Which_ FlutterGoldenFileComparator is |
| 28 | /// instantiated is based on the current testing environment. |
Todd Volkert | 65079ad | 2018-05-03 07:39:41 -0700 | [diff] [blame] | 29 | Future<void> main(FutureOr<void> testMain()) async { |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 30 | const Platform platform = LocalPlatform(); |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 31 | if (FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform)) { |
| 32 | goldenFileComparator = await FlutterPostSubmitFileComparator.fromDefaultComparator(platform); |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 33 | } else if (FlutterPreSubmitFileComparator.isAvailableForEnvironment(platform)) { |
| 34 | goldenFileComparator = await FlutterPreSubmitFileComparator.fromDefaultComparator(platform); |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 35 | } else if (FlutterSkippingFileComparator.isAvailableForEnvironment(platform)) { |
| 36 | goldenFileComparator = FlutterSkippingFileComparator.fromDefaultComparator( |
| 37 | 'Golden file testing is not executed on some Cirrus & Luci environments.' |
Kate Lovett | b1ca7f4 | 2019-11-15 13:04:59 -0800 | [diff] [blame] | 38 | ); |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 39 | } else { |
| 40 | goldenFileComparator = await FlutterLocalFileComparator.fromDefaultComparator(platform); |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 41 | } |
Kate Lovett | b1ca7f4 | 2019-11-15 13:04:59 -0800 | [diff] [blame] | 42 | |
Todd Volkert | 65079ad | 2018-05-03 07:39:41 -0700 | [diff] [blame] | 43 | await testMain(); |
| 44 | } |
| 45 | |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 46 | /// Abstract base class golden file comparator specific to the `flutter/flutter` |
| 47 | /// repository. |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 48 | /// |
| 49 | /// Golden file testing for the `flutter/flutter` repository is handled by three |
| 50 | /// different [FlutterGoldenFileComparator]s, depending on the current testing |
| 51 | /// environment. |
| 52 | /// |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 53 | /// * The [FlutterPostSubmitFileComparator] is utilized during post-submit |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 54 | /// testing, after a pull request has landed on the master branch. This |
| 55 | /// comparator uses the [SkiaGoldClient] and the `goldctl` tool to upload |
| 56 | /// tests to the [Flutter Gold dashboard](https://flutter-gold.skia.org). |
| 57 | /// Flutter Gold manages the master golden files for the `flutter/flutter` |
| 58 | /// repository. |
| 59 | /// |
| 60 | /// * The [FlutterPreSubmitFileComparator] is utilized in pre-submit testing, |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 61 | /// before a pull request lands on the master branch. When authorized, this |
| 62 | /// comparator uses the [SkiaGoldClient] to execute tryjobs, allowing |
| 63 | /// contributors to view and check in visual differences before landing the |
| 64 | /// change. |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 65 | /// |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 66 | /// * When unable to authenticate the `goldctl` tool, this comparator |
| 67 | /// uses the [SkiaGoldClient] to request the baseline images kept by the |
| 68 | /// [Flutter Gold dashboard](https://flutter-gold.skia.org). It then |
| 69 | /// compares the current test image to the baseline images using the |
| 70 | /// standard [GoldenFileComparator.compareLists] to detect any pixel |
| 71 | /// difference. The [SkiaGoldClient] is also used in this case to check |
| 72 | /// the active ignores from the dashboard, in order to allow intended |
| 73 | /// changes to pass tests. |
| 74 | /// |
| 75 | /// * The [FlutterLocalFileComparator] is used for local development testing. |
| 76 | /// Similar to the unauthorized implementation of the |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 77 | /// [FlutterPreSubmitFileComparator], this comparator will use the |
| 78 | /// [SkiaGoldClient] to request baseline images from |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 79 | /// [Flutter Gold](https://flutter-gold.skia.org) and manually compare |
| 80 | /// pixels. If a difference is detected, this comparator will |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 81 | /// generate failure output illustrating the found difference. If a baseline |
| 82 | /// is not found for a given test image, it will consider it a new test and |
| 83 | /// output the new image for verification. |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 84 | /// |
| 85 | /// The [FlutterSkippingFileComparator] is utilized to skip tests outside |
| 86 | /// of the appropriate environments described above. Currently, some Cirrus |
| 87 | /// test shards and Luci environments do not execute golden file testing, and |
| 88 | /// as such do not require a comparator. This comparator is also used when an |
| 89 | /// internet connection is unavailable. |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 90 | abstract class FlutterGoldenFileComparator extends GoldenFileComparator { |
| 91 | /// Creates a [FlutterGoldenFileComparator] that will resolve golden file |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 92 | /// URIs relative to the specified [basedir], and retrieve golden baselines |
| 93 | /// using the [skiaClient]. The [basedir] is used for writing and accessing |
| 94 | /// information and files for interacting with the [skiaClient]. When testing |
| 95 | /// locally, the [basedir] will also contain any diffs from failed tests, or |
| 96 | /// goldens generated from newly introduced tests. |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 97 | /// |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 98 | /// The [fs] and [platform] parameters are useful in tests, where the default |
| 99 | /// file system and platform can be replaced by mock instances. |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 100 | @visibleForTesting |
| 101 | FlutterGoldenFileComparator( |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 102 | this.basedir, |
| 103 | this.skiaClient, { |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 104 | this.fs = const LocalFileSystem(), |
| 105 | this.platform = const LocalPlatform(), |
| 106 | }) : assert(basedir != null), |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 107 | assert(skiaClient != null), |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 108 | assert(fs != null), |
| 109 | assert(platform != null); |
| 110 | |
| 111 | /// The directory to which golden file URIs will be resolved in [compare] and |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 112 | /// [update], cannot be null. |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 113 | final Uri basedir; |
| 114 | |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 115 | /// A client for uploading image tests and making baseline requests to the |
| 116 | /// Flutter Gold Dashboard, cannot be null. |
| 117 | final SkiaGoldClient skiaClient; |
| 118 | |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 119 | /// The file system used to perform file access. |
| 120 | @visibleForTesting |
| 121 | final FileSystem fs; |
| 122 | |
| 123 | /// A wrapper for the [dart:io.Platform] API. |
| 124 | @visibleForTesting |
| 125 | final Platform platform; |
| 126 | |
| 127 | @override |
| 128 | Future<void> update(Uri golden, Uint8List imageBytes) async { |
| 129 | final File goldenFile = getGoldenFile(golden); |
| 130 | await goldenFile.parent.create(recursive: true); |
| 131 | await goldenFile.writeAsBytes(imageBytes, flush: true); |
| 132 | } |
| 133 | |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 134 | @override |
| 135 | Uri getTestUri(Uri key, int version) => key; |
| 136 | |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 137 | /// Calculate the appropriate basedir for the current test context. |
Kate Lovett | 9011cec | 2019-12-04 15:43:36 -0800 | [diff] [blame] | 138 | /// |
| 139 | /// The optional [suffix] argument is used by the |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 140 | /// [FlutterPostSubmitFileComparator] and the [FlutterPreSubmitFileComparator]. |
Kate Lovett | 9011cec | 2019-12-04 15:43:36 -0800 | [diff] [blame] | 141 | /// These [FlutterGoldenFileComparators] randomize their base directories to |
| 142 | /// maintain thread safety while using the `goldctl` tool. |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 143 | @protected |
| 144 | @visibleForTesting |
Kate Lovett | 1c15cd8 | 2020-02-05 11:03:02 -0800 | [diff] [blame] | 145 | static Directory getBaseDirectory( |
| 146 | LocalFileComparator defaultComparator, |
| 147 | Platform platform, { |
| 148 | String suffix = '', |
| 149 | bool local = false, |
| 150 | }) { |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 151 | const FileSystem fs = LocalFileSystem(); |
| 152 | final Directory flutterRoot = fs.directory(platform.environment[_kFlutterRootKey]); |
Kate Lovett | 1c15cd8 | 2020-02-05 11:03:02 -0800 | [diff] [blame] | 153 | Directory comparisonRoot; |
| 154 | |
| 155 | if (!local) { |
| 156 | comparisonRoot = fs.systemTempDirectory.childDirectory( |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 157 | 'skia_goldens_$suffix' |
Kate Lovett | 1c15cd8 | 2020-02-05 11:03:02 -0800 | [diff] [blame] | 158 | ); |
| 159 | } else { |
| 160 | comparisonRoot = flutterRoot.childDirectory( |
| 161 | fs.path.join( |
| 162 | 'bin', |
| 163 | 'cache', |
| 164 | 'pkg', |
| 165 | 'skia_goldens', |
| 166 | ) |
| 167 | ); |
| 168 | } |
| 169 | |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 170 | final Directory testDirectory = fs.directory(defaultComparator.basedir); |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 171 | final String testDirectoryRelativePath = fs.path.relative( |
| 172 | testDirectory.path, |
| 173 | from: flutterRoot.path, |
| 174 | ); |
| 175 | return comparisonRoot.childDirectory(testDirectoryRelativePath); |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 176 | } |
| 177 | |
| 178 | /// Returns the golden [File] identified by the given [Uri]. |
| 179 | @protected |
| 180 | File getGoldenFile(Uri uri) { |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 181 | final File goldenFile = fs.directory(basedir).childFile(fs.file(uri).path); |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 182 | return goldenFile; |
| 183 | } |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 184 | |
Greg Spencer | a60bf8e | 2019-11-22 08:43:55 -0800 | [diff] [blame] | 185 | /// Prepends the golden URL with the library name that encloses the current |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 186 | /// test. |
| 187 | Uri _addPrefix(Uri golden) { |
| 188 | final String prefix = basedir.pathSegments[basedir.pathSegments.length - 2]; |
| 189 | return Uri.parse(prefix + '.' + golden.toString()); |
Todd Volkert | 65079ad | 2018-05-03 07:39:41 -0700 | [diff] [blame] | 190 | } |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 191 | } |
| 192 | |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 193 | /// A [FlutterGoldenFileComparator] for testing golden images with Skia Gold in |
| 194 | /// post-submit. |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 195 | /// |
| 196 | /// For testing across all platforms, the [SkiaGoldClient] is used to upload |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 197 | /// images for framework-related golden tests and process results. |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 198 | /// |
| 199 | /// See also: |
| 200 | /// |
| 201 | /// * [GoldenFileComparator], the abstract class that |
| 202 | /// [FlutterGoldenFileComparator] implements. |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 203 | /// * [FlutterPreSubmitFileComparator], another |
| 204 | /// [FlutterGoldenFileComparator] that tests golden images before changes are |
| 205 | /// merged into the master branch. |
| 206 | /// * [FlutterLocalFileComparator], another |
| 207 | /// [FlutterGoldenFileComparator] that tests golden images locally on your |
| 208 | /// current machine. |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 209 | class FlutterPostSubmitFileComparator extends FlutterGoldenFileComparator { |
| 210 | /// Creates a [FlutterPostSubmitFileComparator] that will test golden file |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 211 | /// images against Skia Gold. |
| 212 | /// |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 213 | /// The [fs] and [platform] parameters are useful in tests, where the default |
| 214 | /// file system and platform can be replaced by mock instances. |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 215 | FlutterPostSubmitFileComparator( |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 216 | final Uri basedir, |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 217 | final SkiaGoldClient skiaClient, { |
| 218 | final FileSystem fs = const LocalFileSystem(), |
| 219 | final Platform platform = const LocalPlatform(), |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 220 | }) : super( |
| 221 | basedir, |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 222 | skiaClient, |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 223 | fs: fs, |
| 224 | platform: platform, |
| 225 | ); |
| 226 | |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 227 | /// Creates a new [FlutterPostSubmitFileComparator] that mirrors the relative |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 228 | /// path resolution of the default [goldenFileComparator]. |
| 229 | /// |
| 230 | /// The [goldens] and [defaultComparator] parameters are visible for testing |
| 231 | /// purposes only. |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 232 | static Future<FlutterPostSubmitFileComparator> fromDefaultComparator( |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 233 | final Platform platform, { |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 234 | SkiaGoldClient goldens, |
| 235 | LocalFileComparator defaultComparator, |
| 236 | }) async { |
Kate Lovett | 8df0d65 | 2019-10-21 16:45:56 -0700 | [diff] [blame] | 237 | |
Alexandre Ardhuin | 1f3ff5c | 2019-11-21 17:46:37 +0100 | [diff] [blame] | 238 | defaultComparator ??= goldenFileComparator as LocalFileComparator; |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 239 | final Directory baseDirectory = FlutterGoldenFileComparator.getBaseDirectory( |
| 240 | defaultComparator, |
| 241 | platform, |
Kate Lovett | 1374a41 | 2019-11-27 11:29:53 -0800 | [diff] [blame] | 242 | suffix: '${math.Random().nextInt(10000)}', |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 243 | ); |
Kate Lovett | 9011cec | 2019-12-04 15:43:36 -0800 | [diff] [blame] | 244 | baseDirectory.createSync(recursive: true); |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 245 | |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 246 | goldens ??= SkiaGoldClient( |
| 247 | baseDirectory, |
| 248 | ci: platform.environment.containsKey('CIRRUS_CI') |
| 249 | ? ContinuousIntegrationEnvironment.cirrus |
| 250 | : ContinuousIntegrationEnvironment.luci, |
| 251 | ); |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 252 | await goldens.auth(); |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 253 | await goldens.imgtestInit(); |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 254 | return FlutterPostSubmitFileComparator(baseDirectory.uri, goldens); |
Todd Volkert | 65079ad | 2018-05-03 07:39:41 -0700 | [diff] [blame] | 255 | } |
Kate Lovett | eb0b179 | 2019-07-12 12:23:04 -0700 | [diff] [blame] | 256 | |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 257 | @override |
| 258 | Future<bool> compare(Uint8List imageBytes, Uri golden) async { |
| 259 | golden = _addPrefix(golden); |
| 260 | await update(golden, imageBytes); |
Kate Lovett | 3a3939a | 2019-10-21 17:31:54 -0700 | [diff] [blame] | 261 | final File goldenFile = getGoldenFile(golden); |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 262 | |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 263 | return skiaClient.imgtestAdd(golden.path, goldenFile); |
| 264 | } |
Kate Lovett | 3a3939a | 2019-10-21 17:31:54 -0700 | [diff] [blame] | 265 | |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 266 | /// Decides based on the current environment if goldens tests should be |
| 267 | /// executed through Skia Gold. |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 268 | static bool isAvailableForEnvironment(Platform platform) { |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 269 | final String cirrusPR = platform.environment['CIRRUS_PR'] ?? ''; |
| 270 | final String cirrusBranch = platform.environment['CIRRUS_BRANCH'] ?? ''; |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 271 | final bool cirrusPostSubmit = platform.environment.containsKey('CIRRUS_CI') |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 272 | && cirrusPR.isEmpty |
| 273 | && cirrusBranch == 'master' |
Kate Lovett | 26d09f1 | 2019-10-31 11:44:07 -0700 | [diff] [blame] | 274 | && platform.environment.containsKey('GOLD_SERVICE_ACCOUNT'); |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 275 | |
| 276 | final bool luciPostSubmit = platform.environment.containsKey('SWARMING_TASK_ID') |
| 277 | && platform.environment.containsKey('GOLDCTL') |
| 278 | // Luci tryjob environments contain this value to inform the [FlutterPreSubmitComparator]. |
| 279 | && !platform.environment.containsKey('GOLD_TRYJOB'); |
| 280 | |
| 281 | return cirrusPostSubmit || luciPostSubmit; |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 282 | } |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 283 | } |
Kate Lovett | 3a3939a | 2019-10-21 17:31:54 -0700 | [diff] [blame] | 284 | |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 285 | /// A [FlutterGoldenFileComparator] for testing golden images before changes are |
| 286 | /// merged into the master branch. |
| 287 | /// |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 288 | /// When authorized (on luci and most cirrus testing conditions), the comparator |
| 289 | /// executes tryjobs using the [SkiaGoldClient]. |
| 290 | /// |
| 291 | /// When unauthorized, this comparator utilizes the [SkiaGoldClient] to request |
| 292 | /// baseline images for the given device under test for manual comparison. |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 293 | /// |
| 294 | /// See also: |
| 295 | /// |
| 296 | /// * [GoldenFileComparator], the abstract class that |
| 297 | /// [FlutterGoldenFileComparator] implements. |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 298 | /// * [FlutterPostSubmitFileComparator], another |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 299 | /// [FlutterGoldenFileComparator] that uploads tests to the Skia Gold |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 300 | /// dashboard in post-submit. |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 301 | /// * [FlutterLocalFileComparator], another |
| 302 | /// [FlutterGoldenFileComparator] that tests golden images locally on your |
| 303 | /// current machine. |
| 304 | class FlutterPreSubmitFileComparator extends FlutterGoldenFileComparator { |
| 305 | /// Creates a [FlutterPreSubmitFileComparator] that will test golden file |
| 306 | /// images against baselines requested from Flutter Gold. |
| 307 | /// |
| 308 | /// The [fs] and [platform] parameters are useful in tests, where the default |
| 309 | /// file system and platform can be replaced by mock instances. |
| 310 | FlutterPreSubmitFileComparator( |
| 311 | final Uri basedir, |
| 312 | final SkiaGoldClient skiaClient, { |
| 313 | final FileSystem fs = const LocalFileSystem(), |
| 314 | final Platform platform = const LocalPlatform(), |
| 315 | }) : super( |
| 316 | basedir, |
| 317 | skiaClient, |
| 318 | fs: fs, |
| 319 | platform: platform, |
| 320 | ); |
| 321 | |
| 322 | /// Creates a new [FlutterPreSubmitFileComparator] that mirrors the |
| 323 | /// relative path resolution of the default [goldenFileComparator]. |
| 324 | /// |
| 325 | /// The [goldens] and [defaultComparator] parameters are visible for testing |
| 326 | /// purposes only. |
| 327 | static Future<FlutterGoldenFileComparator> fromDefaultComparator( |
| 328 | final Platform platform, { |
| 329 | SkiaGoldClient goldens, |
| 330 | LocalFileComparator defaultComparator, |
Kate Lovett | 84aa29c | 2020-01-09 08:08:03 -0800 | [diff] [blame] | 331 | final Directory testBasedir, |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 332 | }) async { |
| 333 | |
Alexandre Ardhuin | 1f3ff5c | 2019-11-21 17:46:37 +0100 | [diff] [blame] | 334 | defaultComparator ??= goldenFileComparator as LocalFileComparator; |
Kate Lovett | 84aa29c | 2020-01-09 08:08:03 -0800 | [diff] [blame] | 335 | final Directory baseDirectory = testBasedir ?? FlutterGoldenFileComparator.getBaseDirectory( |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 336 | defaultComparator, |
| 337 | platform, |
Kate Lovett | 9011cec | 2019-12-04 15:43:36 -0800 | [diff] [blame] | 338 | suffix: '${math.Random().nextInt(10000)}', |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 339 | ); |
Kate Lovett | 84aa29c | 2020-01-09 08:08:03 -0800 | [diff] [blame] | 340 | |
| 341 | if (!baseDirectory.existsSync()) |
| 342 | baseDirectory.createSync(recursive: true); |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 343 | |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 344 | goldens ??= SkiaGoldClient( |
| 345 | baseDirectory, |
| 346 | ci: platform.environment.containsKey('CIRRUS_CI') |
| 347 | ? ContinuousIntegrationEnvironment.cirrus |
| 348 | : ContinuousIntegrationEnvironment.luci, |
| 349 | ); |
Kate Lovett | 84aa29c | 2020-01-09 08:08:03 -0800 | [diff] [blame] | 350 | |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 351 | bool onCirrusWithPermission = false; |
| 352 | if (platform.environment.containsKey('GOLD_SERVICE_ACCOUNT')) { |
| 353 | // Some contributors may not have permission on Cirrus to decrypt the |
| 354 | // service account. |
| 355 | onCirrusWithPermission = |
| 356 | !platform.environment['GOLD_SERVICE_ACCOUNT'].startsWith('ENCRYPTED'); |
| 357 | } |
| 358 | final bool onLuci = platform.environment.containsKey('SWARMING_TASK_ID'); |
| 359 | if (onCirrusWithPermission || onLuci) { |
Kate Lovett | 84aa29c | 2020-01-09 08:08:03 -0800 | [diff] [blame] | 360 | await goldens.auth(); |
| 361 | await goldens.tryjobInit(); |
| 362 | return _AuthorizedFlutterPreSubmitComparator( |
| 363 | baseDirectory.uri, |
| 364 | goldens, |
| 365 | platform: platform, |
| 366 | ); |
| 367 | } |
| 368 | |
| 369 | goldens.emptyAuth(); |
| 370 | return _UnauthorizedFlutterPreSubmitComparator( |
| 371 | baseDirectory.uri, |
| 372 | goldens, |
| 373 | platform: platform, |
| 374 | ); |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 375 | } |
| 376 | |
| 377 | @override |
| 378 | Future<bool> compare(Uint8List imageBytes, Uri golden) async { |
Kate Lovett | 84aa29c | 2020-01-09 08:08:03 -0800 | [diff] [blame] | 379 | assert( |
| 380 | false, |
| 381 | 'The FlutterPreSubmitFileComparator has been used to execute a golden ' |
| 382 | 'file test; this should never happen. Presubmit golden file testing ' |
| 383 | 'should be executed by either the _AuthorizedFlutterPreSubmitComparator ' |
| 384 | 'or the _UnauthorizedFlutterPreSubmitComparator based on contributor ' |
| 385 | 'permissions.' |
| 386 | ); |
| 387 | return false; |
| 388 | } |
| 389 | |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 390 | /// Decides based on the current environment if goldens tests should be |
| 391 | /// executed as pre-submit tests with Skia Gold. |
Kate Lovett | 84aa29c | 2020-01-09 08:08:03 -0800 | [diff] [blame] | 392 | static bool isAvailableForEnvironment(Platform platform) { |
| 393 | final String cirrusPR = platform.environment['CIRRUS_PR'] ?? ''; |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 394 | final bool cirrusPreSubmit = platform.environment.containsKey('CIRRUS_CI') |
Kate Lovett | 84aa29c | 2020-01-09 08:08:03 -0800 | [diff] [blame] | 395 | && cirrusPR.isNotEmpty |
| 396 | && platform.environment.containsKey('GOLD_SERVICE_ACCOUNT'); |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 397 | |
| 398 | final bool luciPreSubmit = platform.environment.containsKey('SWARMING_TASK_ID') |
| 399 | && platform.environment.containsKey('GOLDCTL') |
| 400 | && platform.environment.containsKey('GOLD_TRYJOB'); |
| 401 | return cirrusPreSubmit || luciPreSubmit; |
Kate Lovett | 84aa29c | 2020-01-09 08:08:03 -0800 | [diff] [blame] | 402 | } |
| 403 | } |
| 404 | |
| 405 | class _AuthorizedFlutterPreSubmitComparator extends FlutterPreSubmitFileComparator { |
| 406 | _AuthorizedFlutterPreSubmitComparator( |
| 407 | final Uri basedir, |
| 408 | final SkiaGoldClient skiaClient, { |
| 409 | final FileSystem fs = const LocalFileSystem(), |
| 410 | final Platform platform = const LocalPlatform(), |
| 411 | }) : super( |
| 412 | basedir, |
| 413 | skiaClient, |
| 414 | fs: fs, |
| 415 | platform: platform, |
| 416 | ); |
| 417 | |
| 418 | @override |
| 419 | Future<bool> compare(Uint8List imageBytes, Uri golden) async { |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 420 | golden = _addPrefix(golden); |
Kate Lovett | 9011cec | 2019-12-04 15:43:36 -0800 | [diff] [blame] | 421 | await update(golden, imageBytes); |
| 422 | final File goldenFile = getGoldenFile(golden); |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 423 | |
Kate Lovett | aa622fb | 2020-03-13 11:41:01 -0700 | [diff] [blame] | 424 | await skiaClient.tryjobAdd(golden.path, goldenFile); |
| 425 | |
| 426 | // This will always return true since golden file test failures are managed |
| 427 | // in pre-submit checks by the flutter-gold status check. |
| 428 | return true; |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 429 | } |
Kate Lovett | 84aa29c | 2020-01-09 08:08:03 -0800 | [diff] [blame] | 430 | } |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 431 | |
Kate Lovett | 84aa29c | 2020-01-09 08:08:03 -0800 | [diff] [blame] | 432 | class _UnauthorizedFlutterPreSubmitComparator extends FlutterPreSubmitFileComparator { |
| 433 | _UnauthorizedFlutterPreSubmitComparator( |
| 434 | final Uri basedir, |
| 435 | final SkiaGoldClient skiaClient, { |
| 436 | final FileSystem fs = const LocalFileSystem(), |
| 437 | final Platform platform = const LocalPlatform(), |
| 438 | }) : super( |
| 439 | basedir, |
| 440 | skiaClient, |
| 441 | fs: fs, |
| 442 | platform: platform, |
| 443 | ); |
| 444 | |
| 445 | @override |
| 446 | Future<bool> compare(Uint8List imageBytes, Uri golden) async { |
| 447 | golden = _addPrefix(golden); |
| 448 | await update(golden, imageBytes); |
| 449 | final File goldenFile = getGoldenFile(golden); |
| 450 | |
| 451 | // Check for match to existing baseline. |
| 452 | if (await skiaClient.imgtestCheck(golden.path, goldenFile)) |
| 453 | return true; |
| 454 | |
| 455 | // We do not have a matching image, so we need to check a few things |
| 456 | // manually. We wait until this point to do this work so request traffic |
| 457 | // low. |
| 458 | skiaClient.getExpectations(); |
| 459 | final String testName = skiaClient.cleanTestName(golden.path); |
| 460 | final List<String> testExpectations = skiaClient.expectations[testName]; |
| 461 | if (testExpectations == null) { |
| 462 | // This is a new test. |
| 463 | print('No expectations provided by Skia Gold for test: $golden. ' |
| 464 | 'This may be a new test. If this is an unexpected result, check ' |
| 465 | 'https://flutter-gold.skia.org.\n' |
| 466 | ); |
| 467 | return true; |
| 468 | } |
| 469 | |
| 470 | // Contributors without the proper permissions to execute a tryjob can make |
| 471 | // a golden file change through Gold's ignore feature instead. |
| 472 | final bool ignoreResult = await skiaClient.testIsIgnoredForPullRequest( |
| 473 | platform.environment['CIRRUS_PR'] ?? '', |
| 474 | golden.path, |
| 475 | ); |
| 476 | // If true, this is an intended change. |
| 477 | return ignoreResult; |
Kate Lovett | 26d09f1 | 2019-10-31 11:44:07 -0700 | [diff] [blame] | 478 | } |
| 479 | } |
| 480 | |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 481 | /// A [FlutterGoldenFileComparator] for testing conditions that do not execute |
| 482 | /// golden file tests. |
Kate Lovett | 26d09f1 | 2019-10-31 11:44:07 -0700 | [diff] [blame] | 483 | /// |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 484 | /// Currently, this comparator is used in some Cirrus test shards and Luci |
| 485 | /// environments, as well as when an internet connection is not available for |
| 486 | /// contacting Gold. |
Kate Lovett | 26d09f1 | 2019-10-31 11:44:07 -0700 | [diff] [blame] | 487 | /// |
| 488 | /// See also: |
| 489 | /// |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 490 | /// * [FlutterPostSubmitFileComparator], another [FlutterGoldenFileComparator] |
Kate Lovett | 26d09f1 | 2019-10-31 11:44:07 -0700 | [diff] [blame] | 491 | /// that tests golden images through Skia Gold. |
| 492 | /// * [FlutterPreSubmitFileComparator], another |
| 493 | /// [FlutterGoldenFileComparator] that tests golden images before changes are |
| 494 | /// merged into the master branch. |
| 495 | /// * [FlutterLocalFileComparator], another |
| 496 | /// [FlutterGoldenFileComparator] that tests golden images locally on your |
| 497 | /// current machine. |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 498 | class FlutterSkippingFileComparator extends FlutterGoldenFileComparator { |
| 499 | /// Creates a [FlutterSkippingFileComparator] that will skip tests that |
Kate Lovett | 26d09f1 | 2019-10-31 11:44:07 -0700 | [diff] [blame] | 500 | /// are not in the right environment for golden file testing. |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 501 | FlutterSkippingFileComparator( |
Kate Lovett | 26d09f1 | 2019-10-31 11:44:07 -0700 | [diff] [blame] | 502 | final Uri basedir, |
| 503 | final SkiaGoldClient skiaClient, |
Kate Lovett | b1ca7f4 | 2019-11-15 13:04:59 -0800 | [diff] [blame] | 504 | this.reason, |
| 505 | ) : assert(reason != null), |
| 506 | super(basedir, skiaClient); |
| 507 | |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 508 | /// Describes the reason for using the [FlutterSkippingFileComparator]. |
Kate Lovett | b1ca7f4 | 2019-11-15 13:04:59 -0800 | [diff] [blame] | 509 | /// |
| 510 | /// Cannot be null. |
| 511 | final String reason; |
Kate Lovett | 26d09f1 | 2019-10-31 11:44:07 -0700 | [diff] [blame] | 512 | |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 513 | /// Creates a new [FlutterSkippingFileComparator] that mirrors the |
Kate Lovett | 26d09f1 | 2019-10-31 11:44:07 -0700 | [diff] [blame] | 514 | /// relative path resolution of the default [goldenFileComparator]. |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 515 | static FlutterSkippingFileComparator fromDefaultComparator( |
Kate Lovett | b1ca7f4 | 2019-11-15 13:04:59 -0800 | [diff] [blame] | 516 | String reason, { |
Kate Lovett | 26d09f1 | 2019-10-31 11:44:07 -0700 | [diff] [blame] | 517 | LocalFileComparator defaultComparator, |
| 518 | }) { |
Alexandre Ardhuin | 1f3ff5c | 2019-11-21 17:46:37 +0100 | [diff] [blame] | 519 | defaultComparator ??= goldenFileComparator as LocalFileComparator; |
Kate Lovett | 26d09f1 | 2019-10-31 11:44:07 -0700 | [diff] [blame] | 520 | const FileSystem fs = LocalFileSystem(); |
| 521 | final Uri basedir = defaultComparator.basedir; |
| 522 | final SkiaGoldClient skiaClient = SkiaGoldClient(fs.directory(basedir)); |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 523 | return FlutterSkippingFileComparator(basedir, skiaClient, reason); |
Kate Lovett | 26d09f1 | 2019-10-31 11:44:07 -0700 | [diff] [blame] | 524 | } |
| 525 | |
| 526 | @override |
| 527 | Future<bool> compare(Uint8List imageBytes, Uri golden) async { |
| 528 | print( |
Kate Lovett | b1ca7f4 | 2019-11-15 13:04:59 -0800 | [diff] [blame] | 529 | 'Skipping "$golden" test : $reason' |
Kate Lovett | 26d09f1 | 2019-10-31 11:44:07 -0700 | [diff] [blame] | 530 | ); |
| 531 | return true; |
| 532 | } |
| 533 | |
| 534 | @override |
| 535 | Future<void> update(Uri golden, Uint8List imageBytes) => null; |
| 536 | |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 537 | /// Decides, based on the current environment, if this comparator should be |
Kate Lovett | 26d09f1 | 2019-10-31 11:44:07 -0700 | [diff] [blame] | 538 | /// used. |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 539 | /// |
| 540 | /// If we are in a CI environment, luci or Cirrus, but are not using the other |
| 541 | /// comparators, we skip. |
Kate Lovett | 26d09f1 | 2019-10-31 11:44:07 -0700 | [diff] [blame] | 542 | static bool isAvailableForEnvironment(Platform platform) { |
Kate Lovett | 84aa29c | 2020-01-09 08:08:03 -0800 | [diff] [blame] | 543 | return platform.environment.containsKey('SWARMING_TASK_ID') |
| 544 | || platform.environment.containsKey('CIRRUS_CI'); |
Kate Lovett | 3a3939a | 2019-10-21 17:31:54 -0700 | [diff] [blame] | 545 | } |
Kate Lovett | 616794f | 2019-07-28 12:26:06 -0700 | [diff] [blame] | 546 | } |
| 547 | |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 548 | /// A [FlutterGoldenFileComparator] for testing golden images locally on your |
| 549 | /// current machine. |
| 550 | /// |
| 551 | /// This comparator utilizes the [SkiaGoldClient] to request baseline images for |
Kate Lovett | 26d09f1 | 2019-10-31 11:44:07 -0700 | [diff] [blame] | 552 | /// the given device under test for comparison. This comparator is initialized |
| 553 | /// when conditions for all other [FlutterGoldenFileComparators] have not been |
| 554 | /// met, see the `isAvailableForEnvironment` method for each one listed below. |
| 555 | /// |
| 556 | /// The [FlutterLocalFileComparator] is intended to run on local machines and |
| 557 | /// serve as a smoke test during development. As such, it will not be able to |
| 558 | /// detect unintended changes on environments other than the currently executing |
| 559 | /// machine, until they are tested using the [FlutterPreSubmitFileComparator]. |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 560 | /// |
| 561 | /// See also: |
| 562 | /// |
| 563 | /// * [GoldenFileComparator], the abstract class that |
| 564 | /// [FlutterGoldenFileComparator] implements. |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 565 | /// * [FlutterPostSubmitFileComparator], another |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 566 | /// [FlutterGoldenFileComparator] that uploads tests to the Skia Gold |
| 567 | /// dashboard. |
| 568 | /// * [FlutterPreSubmitFileComparator], another |
| 569 | /// [FlutterGoldenFileComparator] that tests golden images before changes are |
| 570 | /// merged into the master branch. |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 571 | /// * [FlutterSkippingFileComparator], another |
Kate Lovett | 26d09f1 | 2019-10-31 11:44:07 -0700 | [diff] [blame] | 572 | /// [FlutterGoldenFileComparator] that controls post-submit testing |
| 573 | /// conditions that do not execute golden file tests. |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 574 | class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalComparisonOutput { |
| 575 | /// Creates a [FlutterLocalFileComparator] that will test golden file |
| 576 | /// images against baselines requested from Flutter Gold. |
| 577 | /// |
| 578 | /// The [fs] and [platform] parameters are useful in tests, where the default |
| 579 | /// file system and platform can be replaced by mock instances. |
| 580 | FlutterLocalFileComparator( |
| 581 | final Uri basedir, |
| 582 | final SkiaGoldClient skiaClient, { |
| 583 | final FileSystem fs = const LocalFileSystem(), |
| 584 | final Platform platform = const LocalPlatform(), |
| 585 | }) : super( |
| 586 | basedir, |
| 587 | skiaClient, |
| 588 | fs: fs, |
| 589 | platform: platform, |
| 590 | ); |
| 591 | |
| 592 | /// Creates a new [FlutterLocalFileComparator] that mirrors the |
| 593 | /// relative path resolution of the default [goldenFileComparator]. |
| 594 | /// |
Kate Lovett | b1ca7f4 | 2019-11-15 13:04:59 -0800 | [diff] [blame] | 595 | /// The [goldens], [defaultComparator], and [baseDirectory] parameters are |
| 596 | /// visible for testing purposes only. |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 597 | static Future<FlutterGoldenFileComparator> fromDefaultComparator( |
| 598 | final Platform platform, { |
| 599 | SkiaGoldClient goldens, |
| 600 | LocalFileComparator defaultComparator, |
Kate Lovett | b1ca7f4 | 2019-11-15 13:04:59 -0800 | [diff] [blame] | 601 | Directory baseDirectory, |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 602 | }) async { |
Alexandre Ardhuin | 1f3ff5c | 2019-11-21 17:46:37 +0100 | [diff] [blame] | 603 | defaultComparator ??= goldenFileComparator as LocalFileComparator; |
Kate Lovett | b1ca7f4 | 2019-11-15 13:04:59 -0800 | [diff] [blame] | 604 | baseDirectory ??= FlutterGoldenFileComparator.getBaseDirectory( |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 605 | defaultComparator, |
| 606 | platform, |
Kate Lovett | 1c15cd8 | 2020-02-05 11:03:02 -0800 | [diff] [blame] | 607 | local: true, |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 608 | ); |
| 609 | |
| 610 | if(!baseDirectory.existsSync()) { |
| 611 | baseDirectory.createSync(recursive: true); |
| 612 | } |
| 613 | |
| 614 | goldens ??= SkiaGoldClient(baseDirectory); |
Kate Lovett | b1ca7f4 | 2019-11-15 13:04:59 -0800 | [diff] [blame] | 615 | |
| 616 | try { |
| 617 | await goldens.getExpectations(); |
| 618 | } on io.OSError catch (_) { |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 619 | return FlutterSkippingFileComparator( |
Kate Lovett | b1ca7f4 | 2019-11-15 13:04:59 -0800 | [diff] [blame] | 620 | baseDirectory.uri, |
| 621 | goldens, |
Kate Lovett | 6397c02 | 2020-01-10 14:43:01 -0800 | [diff] [blame] | 622 | 'OSError occurred, could not reach Gold. ' |
| 623 | 'Switching to FlutterSkippingGoldenFileComparator.', |
| 624 | ); |
| 625 | } on io.SocketException catch (_) { |
Kate Lovett | ea67a65 | 2020-03-18 12:56:02 -0700 | [diff] [blame] | 626 | return FlutterSkippingFileComparator( |
Kate Lovett | 6397c02 | 2020-01-10 14:43:01 -0800 | [diff] [blame] | 627 | baseDirectory.uri, |
| 628 | goldens, |
| 629 | 'SocketException occurred, could not reach Gold. ' |
| 630 | 'Switching to FlutterSkippingGoldenFileComparator.', |
Kate Lovett | b1ca7f4 | 2019-11-15 13:04:59 -0800 | [diff] [blame] | 631 | ); |
| 632 | } |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 633 | |
| 634 | return FlutterLocalFileComparator(baseDirectory.uri, goldens); |
| 635 | } |
| 636 | |
| 637 | @override |
| 638 | Future<bool> compare(Uint8List imageBytes, Uri golden) async { |
| 639 | golden = _addPrefix(golden); |
| 640 | final String testName = skiaClient.cleanTestName(golden.path); |
| 641 | final List<String> testExpectations = skiaClient.expectations[testName]; |
| 642 | if (testExpectations == null) { |
| 643 | // There is no baseline for this test |
| 644 | print('No expectations provided by Skia Gold for test: $golden. ' |
| 645 | 'This may be a new test. If this is an unexpected result, check ' |
| 646 | 'https://flutter-gold.skia.org.\n' |
| 647 | 'Validate image output found at $basedir' |
| 648 | ); |
| 649 | update(golden, imageBytes); |
| 650 | return true; |
| 651 | } |
| 652 | |
| 653 | ComparisonResult result; |
| 654 | final Map<String, ComparisonResult> failureDiffs = <String, ComparisonResult>{}; |
Alexandre Ardhuin | 4f9b6cf | 2020-01-07 16:32:04 +0100 | [diff] [blame] | 655 | for (final String expectation in testExpectations) { |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 656 | final List<int> goldenBytes = await skiaClient.getImageBytes(expectation); |
| 657 | |
Jacob MacDonald | efdce3c | 2020-04-09 11:21:23 -0700 | [diff] [blame^] | 658 | result = await GoldenFileComparator.compareLists( |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 659 | imageBytes, |
| 660 | goldenBytes, |
| 661 | ); |
| 662 | |
| 663 | if (result.passed) { |
| 664 | return true; |
| 665 | } |
| 666 | failureDiffs[expectation] = result; |
| 667 | } |
Kate Lovett | dfbbfcd | 2019-11-07 13:01:32 -0800 | [diff] [blame] | 668 | |
Alexandre Ardhuin | 4f9b6cf | 2020-01-07 16:32:04 +0100 | [diff] [blame] | 669 | for (final MapEntry<String, ComparisonResult> entry in failureDiffs.entries) { |
Kate Lovett | dfbbfcd | 2019-11-07 13:01:32 -0800 | [diff] [blame] | 670 | if (await skiaClient.isValidDigestForExpectation(entry.key, golden.path)) |
| 671 | generateFailureOutput(entry.value, golden, basedir, key: entry.key); |
| 672 | } |
Kate Lovett | 7bc0203 | 2019-10-25 15:05:21 -0700 | [diff] [blame] | 673 | return false; |
| 674 | } |
| 675 | } |