blob: 739d6d94bc557d04726c04d1e8dd9e630aceb0f1 [file] [log] [blame]
Ian Hickson449f4a62019-11-27 15:04:02 -08001// Copyright 2014 The Flutter Authors. All rights reserved.
Todd Volkert65079ad2018-05-03 07:39:41 -07002// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
Kate Lovett9e3e44e2019-04-19 11:32:16 -07004
Todd Volkert65079ad2018-05-03 07:39:41 -07005import 'dart:async';
Kate Lovettb1ca7f42019-11-15 13:04:59 -08006import 'dart:io' as io;
Kate Lovett1374a412019-11-27 11:29:53 -08007import 'dart:math' as math;
Todd Volkert65079ad2018-05-03 07:39:41 -07008import 'dart:typed_data';
9
Todd Volkert65079ad2018-05-03 07:39:41 -070010import 'package:file/file.dart';
11import 'package:file/local.dart';
12import 'package:flutter_test/flutter_test.dart';
13import 'package:meta/meta.dart';
Kate Lovett616794f2019-07-28 12:26:06 -070014import 'package:platform/platform.dart';
Todd Volkert65079ad2018-05-03 07:39:41 -070015
Kate Lovett616794f2019-07-28 12:26:06 -070016import 'package:flutter_goldens_client/skia_client.dart';
Kate Lovett616794f2019-07-28 12:26:06 -070017export 'package:flutter_goldens_client/skia_client.dart';
Kate Lovett7e542fc2019-04-19 14:33:49 -070018
Kate Lovett7bc02032019-10-25 15:05:21 -070019// If you are here trying to figure out how to use golden files in the Flutter
20// repo itself, consider reading this wiki page:
21// https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package%3Aflutter
22
23const String _kFlutterRootKey = 'FLUTTER_ROOT';
24
Todd Volkert65079ad2018-05-03 07:39:41 -070025/// Main method that can be used in a `flutter_test_config.dart` file to set
26/// [goldenFileComparator] to an instance of [FlutterGoldenFileComparator] that
Kate Lovett616794f2019-07-28 12:26:06 -070027/// works for the current test. _Which_ FlutterGoldenFileComparator is
28/// instantiated is based on the current testing environment.
Todd Volkert65079ad2018-05-03 07:39:41 -070029Future<void> main(FutureOr<void> testMain()) async {
Kate Lovett616794f2019-07-28 12:26:06 -070030 const Platform platform = LocalPlatform();
Kate Lovettea67a652020-03-18 12:56:02 -070031 if (FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform)) {
32 goldenFileComparator = await FlutterPostSubmitFileComparator.fromDefaultComparator(platform);
Kate Lovett7bc02032019-10-25 15:05:21 -070033 } else if (FlutterPreSubmitFileComparator.isAvailableForEnvironment(platform)) {
34 goldenFileComparator = await FlutterPreSubmitFileComparator.fromDefaultComparator(platform);
Kate Lovettea67a652020-03-18 12:56:02 -070035 } else if (FlutterSkippingFileComparator.isAvailableForEnvironment(platform)) {
36 goldenFileComparator = FlutterSkippingFileComparator.fromDefaultComparator(
37 'Golden file testing is not executed on some Cirrus & Luci environments.'
Kate Lovettb1ca7f42019-11-15 13:04:59 -080038 );
Kate Lovett7bc02032019-10-25 15:05:21 -070039 } else {
40 goldenFileComparator = await FlutterLocalFileComparator.fromDefaultComparator(platform);
Kate Lovett616794f2019-07-28 12:26:06 -070041 }
Kate Lovettb1ca7f42019-11-15 13:04:59 -080042
Todd Volkert65079ad2018-05-03 07:39:41 -070043 await testMain();
44}
45
Kate Lovett616794f2019-07-28 12:26:06 -070046/// Abstract base class golden file comparator specific to the `flutter/flutter`
47/// repository.
Kate Lovett7bc02032019-10-25 15:05:21 -070048///
49/// Golden file testing for the `flutter/flutter` repository is handled by three
50/// different [FlutterGoldenFileComparator]s, depending on the current testing
51/// environment.
52///
Kate Lovettea67a652020-03-18 12:56:02 -070053/// * The [FlutterPostSubmitFileComparator] is utilized during post-submit
Kate Lovett7bc02032019-10-25 15:05:21 -070054/// testing, after a pull request has landed on the master branch. This
55/// comparator uses the [SkiaGoldClient] and the `goldctl` tool to upload
56/// tests to the [Flutter Gold dashboard](https://flutter-gold.skia.org).
57/// Flutter Gold manages the master golden files for the `flutter/flutter`
58/// repository.
59///
60/// * The [FlutterPreSubmitFileComparator] is utilized in pre-submit testing,
Kate Lovettea67a652020-03-18 12:56:02 -070061/// before a pull request lands on the master branch. When authorized, this
62/// comparator uses the [SkiaGoldClient] to execute tryjobs, allowing
63/// contributors to view and check in visual differences before landing the
64/// change.
Kate Lovett7bc02032019-10-25 15:05:21 -070065///
Kate Lovettea67a652020-03-18 12:56:02 -070066/// * When unable to authenticate the `goldctl` tool, this comparator
67/// uses the [SkiaGoldClient] to request the baseline images kept by the
68/// [Flutter Gold dashboard](https://flutter-gold.skia.org). It then
69/// compares the current test image to the baseline images using the
70/// standard [GoldenFileComparator.compareLists] to detect any pixel
71/// difference. The [SkiaGoldClient] is also used in this case to check
72/// the active ignores from the dashboard, in order to allow intended
73/// changes to pass tests.
74///
75/// * The [FlutterLocalFileComparator] is used for local development testing.
76/// Similar to the unauthorized implementation of the
Kate Lovett7bc02032019-10-25 15:05:21 -070077/// [FlutterPreSubmitFileComparator], this comparator will use the
78/// [SkiaGoldClient] to request baseline images from
Kate Lovettea67a652020-03-18 12:56:02 -070079/// [Flutter Gold](https://flutter-gold.skia.org) and manually compare
80/// pixels. If a difference is detected, this comparator will
Kate Lovett7bc02032019-10-25 15:05:21 -070081/// generate failure output illustrating the found difference. If a baseline
82/// is not found for a given test image, it will consider it a new test and
83/// output the new image for verification.
Kate Lovettea67a652020-03-18 12:56:02 -070084///
85/// The [FlutterSkippingFileComparator] is utilized to skip tests outside
86/// of the appropriate environments described above. Currently, some Cirrus
87/// test shards and Luci environments do not execute golden file testing, and
88/// as such do not require a comparator. This comparator is also used when an
89/// internet connection is unavailable.
Kate Lovett616794f2019-07-28 12:26:06 -070090abstract class FlutterGoldenFileComparator extends GoldenFileComparator {
91 /// Creates a [FlutterGoldenFileComparator] that will resolve golden file
Kate Lovett7bc02032019-10-25 15:05:21 -070092 /// URIs relative to the specified [basedir], and retrieve golden baselines
93 /// using the [skiaClient]. The [basedir] is used for writing and accessing
94 /// information and files for interacting with the [skiaClient]. When testing
95 /// locally, the [basedir] will also contain any diffs from failed tests, or
96 /// goldens generated from newly introduced tests.
Kate Lovett616794f2019-07-28 12:26:06 -070097 ///
Kate Lovett7bc02032019-10-25 15:05:21 -070098 /// The [fs] and [platform] parameters are useful in tests, where the default
99 /// file system and platform can be replaced by mock instances.
Kate Lovett616794f2019-07-28 12:26:06 -0700100 @visibleForTesting
101 FlutterGoldenFileComparator(
Kate Lovett7bc02032019-10-25 15:05:21 -0700102 this.basedir,
103 this.skiaClient, {
Kate Lovett616794f2019-07-28 12:26:06 -0700104 this.fs = const LocalFileSystem(),
105 this.platform = const LocalPlatform(),
106 }) : assert(basedir != null),
Kate Lovett7bc02032019-10-25 15:05:21 -0700107 assert(skiaClient != null),
Kate Lovett616794f2019-07-28 12:26:06 -0700108 assert(fs != null),
109 assert(platform != null);
110
111 /// The directory to which golden file URIs will be resolved in [compare] and
Kate Lovett7bc02032019-10-25 15:05:21 -0700112 /// [update], cannot be null.
Kate Lovett616794f2019-07-28 12:26:06 -0700113 final Uri basedir;
114
Kate Lovett7bc02032019-10-25 15:05:21 -0700115 /// A client for uploading image tests and making baseline requests to the
116 /// Flutter Gold Dashboard, cannot be null.
117 final SkiaGoldClient skiaClient;
118
Kate Lovett616794f2019-07-28 12:26:06 -0700119 /// The file system used to perform file access.
120 @visibleForTesting
121 final FileSystem fs;
122
123 /// A wrapper for the [dart:io.Platform] API.
124 @visibleForTesting
125 final Platform platform;
126
127 @override
128 Future<void> update(Uri golden, Uint8List imageBytes) async {
129 final File goldenFile = getGoldenFile(golden);
130 await goldenFile.parent.create(recursive: true);
131 await goldenFile.writeAsBytes(imageBytes, flush: true);
132 }
133
Kate Lovett7bc02032019-10-25 15:05:21 -0700134 @override
135 Uri getTestUri(Uri key, int version) => key;
136
Kate Lovett616794f2019-07-28 12:26:06 -0700137 /// Calculate the appropriate basedir for the current test context.
Kate Lovett9011cec2019-12-04 15:43:36 -0800138 ///
139 /// The optional [suffix] argument is used by the
Kate Lovettea67a652020-03-18 12:56:02 -0700140 /// [FlutterPostSubmitFileComparator] and the [FlutterPreSubmitFileComparator].
Kate Lovett9011cec2019-12-04 15:43:36 -0800141 /// These [FlutterGoldenFileComparators] randomize their base directories to
142 /// maintain thread safety while using the `goldctl` tool.
Kate Lovett616794f2019-07-28 12:26:06 -0700143 @protected
144 @visibleForTesting
Kate Lovett1c15cd82020-02-05 11:03:02 -0800145 static Directory getBaseDirectory(
146 LocalFileComparator defaultComparator,
147 Platform platform, {
148 String suffix = '',
149 bool local = false,
150 }) {
Kate Lovett7bc02032019-10-25 15:05:21 -0700151 const FileSystem fs = LocalFileSystem();
152 final Directory flutterRoot = fs.directory(platform.environment[_kFlutterRootKey]);
Kate Lovett1c15cd82020-02-05 11:03:02 -0800153 Directory comparisonRoot;
154
155 if (!local) {
156 comparisonRoot = fs.systemTempDirectory.childDirectory(
Kate Lovettea67a652020-03-18 12:56:02 -0700157 'skia_goldens_$suffix'
Kate Lovett1c15cd82020-02-05 11:03:02 -0800158 );
159 } else {
160 comparisonRoot = flutterRoot.childDirectory(
161 fs.path.join(
162 'bin',
163 'cache',
164 'pkg',
165 'skia_goldens',
166 )
167 );
168 }
169
Kate Lovett616794f2019-07-28 12:26:06 -0700170 final Directory testDirectory = fs.directory(defaultComparator.basedir);
Kate Lovett7bc02032019-10-25 15:05:21 -0700171 final String testDirectoryRelativePath = fs.path.relative(
172 testDirectory.path,
173 from: flutterRoot.path,
174 );
175 return comparisonRoot.childDirectory(testDirectoryRelativePath);
Kate Lovett616794f2019-07-28 12:26:06 -0700176 }
177
178 /// Returns the golden [File] identified by the given [Uri].
179 @protected
180 File getGoldenFile(Uri uri) {
Kate Lovett616794f2019-07-28 12:26:06 -0700181 final File goldenFile = fs.directory(basedir).childFile(fs.file(uri).path);
Kate Lovett616794f2019-07-28 12:26:06 -0700182 return goldenFile;
183 }
Kate Lovett616794f2019-07-28 12:26:06 -0700184
Greg Spencera60bf8e2019-11-22 08:43:55 -0800185 /// Prepends the golden URL with the library name that encloses the current
Kate Lovett7bc02032019-10-25 15:05:21 -0700186 /// test.
187 Uri _addPrefix(Uri golden) {
188 final String prefix = basedir.pathSegments[basedir.pathSegments.length - 2];
189 return Uri.parse(prefix + '.' + golden.toString());
Todd Volkert65079ad2018-05-03 07:39:41 -0700190 }
Kate Lovett616794f2019-07-28 12:26:06 -0700191}
192
Kate Lovettea67a652020-03-18 12:56:02 -0700193/// A [FlutterGoldenFileComparator] for testing golden images with Skia Gold in
194/// post-submit.
Kate Lovett616794f2019-07-28 12:26:06 -0700195///
196/// For testing across all platforms, the [SkiaGoldClient] is used to upload
Kate Lovettea67a652020-03-18 12:56:02 -0700197/// images for framework-related golden tests and process results.
Kate Lovett616794f2019-07-28 12:26:06 -0700198///
199/// See also:
200///
201/// * [GoldenFileComparator], the abstract class that
202/// [FlutterGoldenFileComparator] implements.
Kate Lovett7bc02032019-10-25 15:05:21 -0700203/// * [FlutterPreSubmitFileComparator], another
204/// [FlutterGoldenFileComparator] that tests golden images before changes are
205/// merged into the master branch.
206/// * [FlutterLocalFileComparator], another
207/// [FlutterGoldenFileComparator] that tests golden images locally on your
208/// current machine.
Kate Lovettea67a652020-03-18 12:56:02 -0700209class FlutterPostSubmitFileComparator extends FlutterGoldenFileComparator {
210 /// Creates a [FlutterPostSubmitFileComparator] that will test golden file
Kate Lovett616794f2019-07-28 12:26:06 -0700211 /// images against Skia Gold.
212 ///
Kate Lovett7bc02032019-10-25 15:05:21 -0700213 /// The [fs] and [platform] parameters are useful in tests, where the default
214 /// file system and platform can be replaced by mock instances.
Kate Lovettea67a652020-03-18 12:56:02 -0700215 FlutterPostSubmitFileComparator(
Kate Lovett616794f2019-07-28 12:26:06 -0700216 final Uri basedir,
Kate Lovett7bc02032019-10-25 15:05:21 -0700217 final SkiaGoldClient skiaClient, {
218 final FileSystem fs = const LocalFileSystem(),
219 final Platform platform = const LocalPlatform(),
Kate Lovett616794f2019-07-28 12:26:06 -0700220 }) : super(
221 basedir,
Kate Lovett7bc02032019-10-25 15:05:21 -0700222 skiaClient,
Kate Lovett616794f2019-07-28 12:26:06 -0700223 fs: fs,
224 platform: platform,
225 );
226
Kate Lovettea67a652020-03-18 12:56:02 -0700227 /// Creates a new [FlutterPostSubmitFileComparator] that mirrors the relative
Kate Lovett616794f2019-07-28 12:26:06 -0700228 /// path resolution of the default [goldenFileComparator].
229 ///
230 /// The [goldens] and [defaultComparator] parameters are visible for testing
231 /// purposes only.
Kate Lovettea67a652020-03-18 12:56:02 -0700232 static Future<FlutterPostSubmitFileComparator> fromDefaultComparator(
Kate Lovett7bc02032019-10-25 15:05:21 -0700233 final Platform platform, {
Kate Lovett616794f2019-07-28 12:26:06 -0700234 SkiaGoldClient goldens,
235 LocalFileComparator defaultComparator,
236 }) async {
Kate Lovett8df0d652019-10-21 16:45:56 -0700237
Alexandre Ardhuin1f3ff5c2019-11-21 17:46:37 +0100238 defaultComparator ??= goldenFileComparator as LocalFileComparator;
Kate Lovett7bc02032019-10-25 15:05:21 -0700239 final Directory baseDirectory = FlutterGoldenFileComparator.getBaseDirectory(
240 defaultComparator,
241 platform,
Kate Lovett1374a412019-11-27 11:29:53 -0800242 suffix: '${math.Random().nextInt(10000)}',
Kate Lovett7bc02032019-10-25 15:05:21 -0700243 );
Kate Lovett9011cec2019-12-04 15:43:36 -0800244 baseDirectory.createSync(recursive: true);
Kate Lovett7bc02032019-10-25 15:05:21 -0700245
Kate Lovettea67a652020-03-18 12:56:02 -0700246 goldens ??= SkiaGoldClient(
247 baseDirectory,
248 ci: platform.environment.containsKey('CIRRUS_CI')
249 ? ContinuousIntegrationEnvironment.cirrus
250 : ContinuousIntegrationEnvironment.luci,
251 );
Kate Lovett7bc02032019-10-25 15:05:21 -0700252 await goldens.auth();
Kate Lovett616794f2019-07-28 12:26:06 -0700253 await goldens.imgtestInit();
Kate Lovettea67a652020-03-18 12:56:02 -0700254 return FlutterPostSubmitFileComparator(baseDirectory.uri, goldens);
Todd Volkert65079ad2018-05-03 07:39:41 -0700255 }
Kate Lovetteb0b1792019-07-12 12:23:04 -0700256
Kate Lovett616794f2019-07-28 12:26:06 -0700257 @override
258 Future<bool> compare(Uint8List imageBytes, Uri golden) async {
259 golden = _addPrefix(golden);
260 await update(golden, imageBytes);
Kate Lovett3a3939a2019-10-21 17:31:54 -0700261 final File goldenFile = getGoldenFile(golden);
Kate Lovett616794f2019-07-28 12:26:06 -0700262
Kate Lovett7bc02032019-10-25 15:05:21 -0700263 return skiaClient.imgtestAdd(golden.path, goldenFile);
264 }
Kate Lovett3a3939a2019-10-21 17:31:54 -0700265
Kate Lovettea67a652020-03-18 12:56:02 -0700266 /// Decides based on the current environment if goldens tests should be
267 /// executed through Skia Gold.
Kate Lovett7bc02032019-10-25 15:05:21 -0700268 static bool isAvailableForEnvironment(Platform platform) {
Kate Lovett616794f2019-07-28 12:26:06 -0700269 final String cirrusPR = platform.environment['CIRRUS_PR'] ?? '';
270 final String cirrusBranch = platform.environment['CIRRUS_BRANCH'] ?? '';
Kate Lovettea67a652020-03-18 12:56:02 -0700271 final bool cirrusPostSubmit = platform.environment.containsKey('CIRRUS_CI')
Kate Lovett616794f2019-07-28 12:26:06 -0700272 && cirrusPR.isEmpty
273 && cirrusBranch == 'master'
Kate Lovett26d09f12019-10-31 11:44:07 -0700274 && platform.environment.containsKey('GOLD_SERVICE_ACCOUNT');
Kate Lovettea67a652020-03-18 12:56:02 -0700275
276 final bool luciPostSubmit = platform.environment.containsKey('SWARMING_TASK_ID')
277 && platform.environment.containsKey('GOLDCTL')
278 // Luci tryjob environments contain this value to inform the [FlutterPreSubmitComparator].
279 && !platform.environment.containsKey('GOLD_TRYJOB');
280
281 return cirrusPostSubmit || luciPostSubmit;
Kate Lovett616794f2019-07-28 12:26:06 -0700282 }
Kate Lovett7bc02032019-10-25 15:05:21 -0700283}
Kate Lovett3a3939a2019-10-21 17:31:54 -0700284
Kate Lovett7bc02032019-10-25 15:05:21 -0700285/// A [FlutterGoldenFileComparator] for testing golden images before changes are
286/// merged into the master branch.
287///
Kate Lovettea67a652020-03-18 12:56:02 -0700288/// When authorized (on luci and most cirrus testing conditions), the comparator
289/// executes tryjobs using the [SkiaGoldClient].
290///
291/// When unauthorized, this comparator utilizes the [SkiaGoldClient] to request
292/// baseline images for the given device under test for manual comparison.
Kate Lovett7bc02032019-10-25 15:05:21 -0700293///
294/// See also:
295///
296/// * [GoldenFileComparator], the abstract class that
297/// [FlutterGoldenFileComparator] implements.
Kate Lovettea67a652020-03-18 12:56:02 -0700298/// * [FlutterPostSubmitFileComparator], another
Kate Lovett7bc02032019-10-25 15:05:21 -0700299/// [FlutterGoldenFileComparator] that uploads tests to the Skia Gold
Kate Lovettea67a652020-03-18 12:56:02 -0700300/// dashboard in post-submit.
Kate Lovett7bc02032019-10-25 15:05:21 -0700301/// * [FlutterLocalFileComparator], another
302/// [FlutterGoldenFileComparator] that tests golden images locally on your
303/// current machine.
304class FlutterPreSubmitFileComparator extends FlutterGoldenFileComparator {
305 /// Creates a [FlutterPreSubmitFileComparator] that will test golden file
306 /// images against baselines requested from Flutter Gold.
307 ///
308 /// The [fs] and [platform] parameters are useful in tests, where the default
309 /// file system and platform can be replaced by mock instances.
310 FlutterPreSubmitFileComparator(
311 final Uri basedir,
312 final SkiaGoldClient skiaClient, {
313 final FileSystem fs = const LocalFileSystem(),
314 final Platform platform = const LocalPlatform(),
315 }) : super(
316 basedir,
317 skiaClient,
318 fs: fs,
319 platform: platform,
320 );
321
322 /// Creates a new [FlutterPreSubmitFileComparator] that mirrors the
323 /// relative path resolution of the default [goldenFileComparator].
324 ///
325 /// The [goldens] and [defaultComparator] parameters are visible for testing
326 /// purposes only.
327 static Future<FlutterGoldenFileComparator> fromDefaultComparator(
328 final Platform platform, {
329 SkiaGoldClient goldens,
330 LocalFileComparator defaultComparator,
Kate Lovett84aa29c2020-01-09 08:08:03 -0800331 final Directory testBasedir,
Kate Lovett7bc02032019-10-25 15:05:21 -0700332 }) async {
333
Alexandre Ardhuin1f3ff5c2019-11-21 17:46:37 +0100334 defaultComparator ??= goldenFileComparator as LocalFileComparator;
Kate Lovett84aa29c2020-01-09 08:08:03 -0800335 final Directory baseDirectory = testBasedir ?? FlutterGoldenFileComparator.getBaseDirectory(
Kate Lovett7bc02032019-10-25 15:05:21 -0700336 defaultComparator,
337 platform,
Kate Lovett9011cec2019-12-04 15:43:36 -0800338 suffix: '${math.Random().nextInt(10000)}',
Kate Lovett7bc02032019-10-25 15:05:21 -0700339 );
Kate Lovett84aa29c2020-01-09 08:08:03 -0800340
341 if (!baseDirectory.existsSync())
342 baseDirectory.createSync(recursive: true);
Kate Lovett7bc02032019-10-25 15:05:21 -0700343
Kate Lovettea67a652020-03-18 12:56:02 -0700344 goldens ??= SkiaGoldClient(
345 baseDirectory,
346 ci: platform.environment.containsKey('CIRRUS_CI')
347 ? ContinuousIntegrationEnvironment.cirrus
348 : ContinuousIntegrationEnvironment.luci,
349 );
Kate Lovett84aa29c2020-01-09 08:08:03 -0800350
Kate Lovettea67a652020-03-18 12:56:02 -0700351 bool onCirrusWithPermission = false;
352 if (platform.environment.containsKey('GOLD_SERVICE_ACCOUNT')) {
353 // Some contributors may not have permission on Cirrus to decrypt the
354 // service account.
355 onCirrusWithPermission =
356 !platform.environment['GOLD_SERVICE_ACCOUNT'].startsWith('ENCRYPTED');
357 }
358 final bool onLuci = platform.environment.containsKey('SWARMING_TASK_ID');
359 if (onCirrusWithPermission || onLuci) {
Kate Lovett84aa29c2020-01-09 08:08:03 -0800360 await goldens.auth();
361 await goldens.tryjobInit();
362 return _AuthorizedFlutterPreSubmitComparator(
363 baseDirectory.uri,
364 goldens,
365 platform: platform,
366 );
367 }
368
369 goldens.emptyAuth();
370 return _UnauthorizedFlutterPreSubmitComparator(
371 baseDirectory.uri,
372 goldens,
373 platform: platform,
374 );
Kate Lovett7bc02032019-10-25 15:05:21 -0700375 }
376
377 @override
378 Future<bool> compare(Uint8List imageBytes, Uri golden) async {
Kate Lovett84aa29c2020-01-09 08:08:03 -0800379 assert(
380 false,
381 'The FlutterPreSubmitFileComparator has been used to execute a golden '
382 'file test; this should never happen. Presubmit golden file testing '
383 'should be executed by either the _AuthorizedFlutterPreSubmitComparator '
384 'or the _UnauthorizedFlutterPreSubmitComparator based on contributor '
385 'permissions.'
386 );
387 return false;
388 }
389
Kate Lovettea67a652020-03-18 12:56:02 -0700390 /// Decides based on the current environment if goldens tests should be
391 /// executed as pre-submit tests with Skia Gold.
Kate Lovett84aa29c2020-01-09 08:08:03 -0800392 static bool isAvailableForEnvironment(Platform platform) {
393 final String cirrusPR = platform.environment['CIRRUS_PR'] ?? '';
Kate Lovettea67a652020-03-18 12:56:02 -0700394 final bool cirrusPreSubmit = platform.environment.containsKey('CIRRUS_CI')
Kate Lovett84aa29c2020-01-09 08:08:03 -0800395 && cirrusPR.isNotEmpty
396 && platform.environment.containsKey('GOLD_SERVICE_ACCOUNT');
Kate Lovettea67a652020-03-18 12:56:02 -0700397
398 final bool luciPreSubmit = platform.environment.containsKey('SWARMING_TASK_ID')
399 && platform.environment.containsKey('GOLDCTL')
400 && platform.environment.containsKey('GOLD_TRYJOB');
401 return cirrusPreSubmit || luciPreSubmit;
Kate Lovett84aa29c2020-01-09 08:08:03 -0800402 }
403}
404
405class _AuthorizedFlutterPreSubmitComparator extends FlutterPreSubmitFileComparator {
406 _AuthorizedFlutterPreSubmitComparator(
407 final Uri basedir,
408 final SkiaGoldClient skiaClient, {
409 final FileSystem fs = const LocalFileSystem(),
410 final Platform platform = const LocalPlatform(),
411 }) : super(
412 basedir,
413 skiaClient,
414 fs: fs,
415 platform: platform,
416 );
417
418 @override
419 Future<bool> compare(Uint8List imageBytes, Uri golden) async {
Kate Lovett7bc02032019-10-25 15:05:21 -0700420 golden = _addPrefix(golden);
Kate Lovett9011cec2019-12-04 15:43:36 -0800421 await update(golden, imageBytes);
422 final File goldenFile = getGoldenFile(golden);
Kate Lovett7bc02032019-10-25 15:05:21 -0700423
Kate Lovettaa622fb2020-03-13 11:41:01 -0700424 await skiaClient.tryjobAdd(golden.path, goldenFile);
425
426 // This will always return true since golden file test failures are managed
427 // in pre-submit checks by the flutter-gold status check.
428 return true;
Kate Lovett7bc02032019-10-25 15:05:21 -0700429 }
Kate Lovett84aa29c2020-01-09 08:08:03 -0800430}
Kate Lovett7bc02032019-10-25 15:05:21 -0700431
Kate Lovett84aa29c2020-01-09 08:08:03 -0800432class _UnauthorizedFlutterPreSubmitComparator extends FlutterPreSubmitFileComparator {
433 _UnauthorizedFlutterPreSubmitComparator(
434 final Uri basedir,
435 final SkiaGoldClient skiaClient, {
436 final FileSystem fs = const LocalFileSystem(),
437 final Platform platform = const LocalPlatform(),
438 }) : super(
439 basedir,
440 skiaClient,
441 fs: fs,
442 platform: platform,
443 );
444
445 @override
446 Future<bool> compare(Uint8List imageBytes, Uri golden) async {
447 golden = _addPrefix(golden);
448 await update(golden, imageBytes);
449 final File goldenFile = getGoldenFile(golden);
450
451 // Check for match to existing baseline.
452 if (await skiaClient.imgtestCheck(golden.path, goldenFile))
453 return true;
454
455 // We do not have a matching image, so we need to check a few things
456 // manually. We wait until this point to do this work so request traffic
457 // low.
458 skiaClient.getExpectations();
459 final String testName = skiaClient.cleanTestName(golden.path);
460 final List<String> testExpectations = skiaClient.expectations[testName];
461 if (testExpectations == null) {
462 // This is a new test.
463 print('No expectations provided by Skia Gold for test: $golden. '
464 'This may be a new test. If this is an unexpected result, check '
465 'https://flutter-gold.skia.org.\n'
466 );
467 return true;
468 }
469
470 // Contributors without the proper permissions to execute a tryjob can make
471 // a golden file change through Gold's ignore feature instead.
472 final bool ignoreResult = await skiaClient.testIsIgnoredForPullRequest(
473 platform.environment['CIRRUS_PR'] ?? '',
474 golden.path,
475 );
476 // If true, this is an intended change.
477 return ignoreResult;
Kate Lovett26d09f12019-10-31 11:44:07 -0700478 }
479}
480
Kate Lovettea67a652020-03-18 12:56:02 -0700481/// A [FlutterGoldenFileComparator] for testing conditions that do not execute
482/// golden file tests.
Kate Lovett26d09f12019-10-31 11:44:07 -0700483///
Kate Lovettea67a652020-03-18 12:56:02 -0700484/// Currently, this comparator is used in some Cirrus test shards and Luci
485/// environments, as well as when an internet connection is not available for
486/// contacting Gold.
Kate Lovett26d09f12019-10-31 11:44:07 -0700487///
488/// See also:
489///
Kate Lovettea67a652020-03-18 12:56:02 -0700490/// * [FlutterPostSubmitFileComparator], another [FlutterGoldenFileComparator]
Kate Lovett26d09f12019-10-31 11:44:07 -0700491/// that tests golden images through Skia Gold.
492/// * [FlutterPreSubmitFileComparator], another
493/// [FlutterGoldenFileComparator] that tests golden images before changes are
494/// merged into the master branch.
495/// * [FlutterLocalFileComparator], another
496/// [FlutterGoldenFileComparator] that tests golden images locally on your
497/// current machine.
Kate Lovettea67a652020-03-18 12:56:02 -0700498class FlutterSkippingFileComparator extends FlutterGoldenFileComparator {
499 /// Creates a [FlutterSkippingFileComparator] that will skip tests that
Kate Lovett26d09f12019-10-31 11:44:07 -0700500 /// are not in the right environment for golden file testing.
Kate Lovettea67a652020-03-18 12:56:02 -0700501 FlutterSkippingFileComparator(
Kate Lovett26d09f12019-10-31 11:44:07 -0700502 final Uri basedir,
503 final SkiaGoldClient skiaClient,
Kate Lovettb1ca7f42019-11-15 13:04:59 -0800504 this.reason,
505 ) : assert(reason != null),
506 super(basedir, skiaClient);
507
Kate Lovettea67a652020-03-18 12:56:02 -0700508 /// Describes the reason for using the [FlutterSkippingFileComparator].
Kate Lovettb1ca7f42019-11-15 13:04:59 -0800509 ///
510 /// Cannot be null.
511 final String reason;
Kate Lovett26d09f12019-10-31 11:44:07 -0700512
Kate Lovettea67a652020-03-18 12:56:02 -0700513 /// Creates a new [FlutterSkippingFileComparator] that mirrors the
Kate Lovett26d09f12019-10-31 11:44:07 -0700514 /// relative path resolution of the default [goldenFileComparator].
Kate Lovettea67a652020-03-18 12:56:02 -0700515 static FlutterSkippingFileComparator fromDefaultComparator(
Kate Lovettb1ca7f42019-11-15 13:04:59 -0800516 String reason, {
Kate Lovett26d09f12019-10-31 11:44:07 -0700517 LocalFileComparator defaultComparator,
518 }) {
Alexandre Ardhuin1f3ff5c2019-11-21 17:46:37 +0100519 defaultComparator ??= goldenFileComparator as LocalFileComparator;
Kate Lovett26d09f12019-10-31 11:44:07 -0700520 const FileSystem fs = LocalFileSystem();
521 final Uri basedir = defaultComparator.basedir;
522 final SkiaGoldClient skiaClient = SkiaGoldClient(fs.directory(basedir));
Kate Lovettea67a652020-03-18 12:56:02 -0700523 return FlutterSkippingFileComparator(basedir, skiaClient, reason);
Kate Lovett26d09f12019-10-31 11:44:07 -0700524 }
525
526 @override
527 Future<bool> compare(Uint8List imageBytes, Uri golden) async {
528 print(
Kate Lovettb1ca7f42019-11-15 13:04:59 -0800529 'Skipping "$golden" test : $reason'
Kate Lovett26d09f12019-10-31 11:44:07 -0700530 );
531 return true;
532 }
533
534 @override
535 Future<void> update(Uri golden, Uint8List imageBytes) => null;
536
Kate Lovettea67a652020-03-18 12:56:02 -0700537 /// Decides, based on the current environment, if this comparator should be
Kate Lovett26d09f12019-10-31 11:44:07 -0700538 /// used.
Kate Lovettea67a652020-03-18 12:56:02 -0700539 ///
540 /// If we are in a CI environment, luci or Cirrus, but are not using the other
541 /// comparators, we skip.
Kate Lovett26d09f12019-10-31 11:44:07 -0700542 static bool isAvailableForEnvironment(Platform platform) {
Kate Lovett84aa29c2020-01-09 08:08:03 -0800543 return platform.environment.containsKey('SWARMING_TASK_ID')
544 || platform.environment.containsKey('CIRRUS_CI');
Kate Lovett3a3939a2019-10-21 17:31:54 -0700545 }
Kate Lovett616794f2019-07-28 12:26:06 -0700546}
547
Kate Lovett7bc02032019-10-25 15:05:21 -0700548/// A [FlutterGoldenFileComparator] for testing golden images locally on your
549/// current machine.
550///
551/// This comparator utilizes the [SkiaGoldClient] to request baseline images for
Kate Lovett26d09f12019-10-31 11:44:07 -0700552/// the given device under test for comparison. This comparator is initialized
553/// when conditions for all other [FlutterGoldenFileComparators] have not been
554/// met, see the `isAvailableForEnvironment` method for each one listed below.
555///
556/// The [FlutterLocalFileComparator] is intended to run on local machines and
557/// serve as a smoke test during development. As such, it will not be able to
558/// detect unintended changes on environments other than the currently executing
559/// machine, until they are tested using the [FlutterPreSubmitFileComparator].
Kate Lovett7bc02032019-10-25 15:05:21 -0700560///
561/// See also:
562///
563/// * [GoldenFileComparator], the abstract class that
564/// [FlutterGoldenFileComparator] implements.
Kate Lovettea67a652020-03-18 12:56:02 -0700565/// * [FlutterPostSubmitFileComparator], another
Kate Lovett7bc02032019-10-25 15:05:21 -0700566/// [FlutterGoldenFileComparator] that uploads tests to the Skia Gold
567/// dashboard.
568/// * [FlutterPreSubmitFileComparator], another
569/// [FlutterGoldenFileComparator] that tests golden images before changes are
570/// merged into the master branch.
Kate Lovettea67a652020-03-18 12:56:02 -0700571/// * [FlutterSkippingFileComparator], another
Kate Lovett26d09f12019-10-31 11:44:07 -0700572/// [FlutterGoldenFileComparator] that controls post-submit testing
573/// conditions that do not execute golden file tests.
Kate Lovett7bc02032019-10-25 15:05:21 -0700574class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalComparisonOutput {
575 /// Creates a [FlutterLocalFileComparator] that will test golden file
576 /// images against baselines requested from Flutter Gold.
577 ///
578 /// The [fs] and [platform] parameters are useful in tests, where the default
579 /// file system and platform can be replaced by mock instances.
580 FlutterLocalFileComparator(
581 final Uri basedir,
582 final SkiaGoldClient skiaClient, {
583 final FileSystem fs = const LocalFileSystem(),
584 final Platform platform = const LocalPlatform(),
585 }) : super(
586 basedir,
587 skiaClient,
588 fs: fs,
589 platform: platform,
590 );
591
592 /// Creates a new [FlutterLocalFileComparator] that mirrors the
593 /// relative path resolution of the default [goldenFileComparator].
594 ///
Kate Lovettb1ca7f42019-11-15 13:04:59 -0800595 /// The [goldens], [defaultComparator], and [baseDirectory] parameters are
596 /// visible for testing purposes only.
Kate Lovett7bc02032019-10-25 15:05:21 -0700597 static Future<FlutterGoldenFileComparator> fromDefaultComparator(
598 final Platform platform, {
599 SkiaGoldClient goldens,
600 LocalFileComparator defaultComparator,
Kate Lovettb1ca7f42019-11-15 13:04:59 -0800601 Directory baseDirectory,
Kate Lovett7bc02032019-10-25 15:05:21 -0700602 }) async {
Alexandre Ardhuin1f3ff5c2019-11-21 17:46:37 +0100603 defaultComparator ??= goldenFileComparator as LocalFileComparator;
Kate Lovettb1ca7f42019-11-15 13:04:59 -0800604 baseDirectory ??= FlutterGoldenFileComparator.getBaseDirectory(
Kate Lovett7bc02032019-10-25 15:05:21 -0700605 defaultComparator,
606 platform,
Kate Lovett1c15cd82020-02-05 11:03:02 -0800607 local: true,
Kate Lovett7bc02032019-10-25 15:05:21 -0700608 );
609
610 if(!baseDirectory.existsSync()) {
611 baseDirectory.createSync(recursive: true);
612 }
613
614 goldens ??= SkiaGoldClient(baseDirectory);
Kate Lovettb1ca7f42019-11-15 13:04:59 -0800615
616 try {
617 await goldens.getExpectations();
618 } on io.OSError catch (_) {
Kate Lovettea67a652020-03-18 12:56:02 -0700619 return FlutterSkippingFileComparator(
Kate Lovettb1ca7f42019-11-15 13:04:59 -0800620 baseDirectory.uri,
621 goldens,
Kate Lovett6397c022020-01-10 14:43:01 -0800622 'OSError occurred, could not reach Gold. '
623 'Switching to FlutterSkippingGoldenFileComparator.',
624 );
625 } on io.SocketException catch (_) {
Kate Lovettea67a652020-03-18 12:56:02 -0700626 return FlutterSkippingFileComparator(
Kate Lovett6397c022020-01-10 14:43:01 -0800627 baseDirectory.uri,
628 goldens,
629 'SocketException occurred, could not reach Gold. '
630 'Switching to FlutterSkippingGoldenFileComparator.',
Kate Lovettb1ca7f42019-11-15 13:04:59 -0800631 );
632 }
Kate Lovett7bc02032019-10-25 15:05:21 -0700633
634 return FlutterLocalFileComparator(baseDirectory.uri, goldens);
635 }
636
637 @override
638 Future<bool> compare(Uint8List imageBytes, Uri golden) async {
639 golden = _addPrefix(golden);
640 final String testName = skiaClient.cleanTestName(golden.path);
641 final List<String> testExpectations = skiaClient.expectations[testName];
642 if (testExpectations == null) {
643 // There is no baseline for this test
644 print('No expectations provided by Skia Gold for test: $golden. '
645 'This may be a new test. If this is an unexpected result, check '
646 'https://flutter-gold.skia.org.\n'
647 'Validate image output found at $basedir'
648 );
649 update(golden, imageBytes);
650 return true;
651 }
652
653 ComparisonResult result;
654 final Map<String, ComparisonResult> failureDiffs = <String, ComparisonResult>{};
Alexandre Ardhuin4f9b6cf2020-01-07 16:32:04 +0100655 for (final String expectation in testExpectations) {
Kate Lovett7bc02032019-10-25 15:05:21 -0700656 final List<int> goldenBytes = await skiaClient.getImageBytes(expectation);
657
Jacob MacDonaldefdce3c2020-04-09 11:21:23 -0700658 result = await GoldenFileComparator.compareLists(
Kate Lovett7bc02032019-10-25 15:05:21 -0700659 imageBytes,
660 goldenBytes,
661 );
662
663 if (result.passed) {
664 return true;
665 }
666 failureDiffs[expectation] = result;
667 }
Kate Lovettdfbbfcd2019-11-07 13:01:32 -0800668
Alexandre Ardhuin4f9b6cf2020-01-07 16:32:04 +0100669 for (final MapEntry<String, ComparisonResult> entry in failureDiffs.entries) {
Kate Lovettdfbbfcd2019-11-07 13:01:32 -0800670 if (await skiaClient.isValidDigestForExpectation(entry.key, golden.path))
671 generateFailureOutput(entry.value, golden, basedir, key: entry.key);
672 }
Kate Lovett7bc02032019-10-25 15:05:21 -0700673 return false;
674 }
675}