commit 53cba24de3a09761b66f367f0dd1f6e3390dac88
author Jenn Magder <magder@google.com> Thu Apr 11 21:07:19 2024 -0700
committer GitHub <noreply@github.com> Fri Apr 12 04:07:19 2024 +0000
tree a9a0a7cd62d19402ad78e24186d1879af57d1f9d
parent a34086a573b79c5253310133a16fe25e08669c88

Allow A/B tests to run as just aggregator of local engine benchmarks (#146479)

Add `--ab-local-engine-only` flag to change A/B benchmark aggregate behavior to instead run the number of tasks specified with the local engine only, not the default engine.  This is useful when you are trying to get average and noise results for a particular engine branch, without needing the default engine every time you change the engine.

Fixes https://github.com/flutter/flutter/issues/143770

dev/devicelab/bin/run.dart

diff --git a/dev/devicelab/bin/run.dart b/dev/devicelab/bin/run.dart
index 628b16f..af5268d 100644
--- a/dev/devicelab/bin/run.dart
+++ b/dev/devicelab/bin/run.dart
@@ -114,6 +114,7 @@
       deviceId: deviceId,
       resultsFile: resultsFile,
       taskName: taskNames.single,
+      onlyLocalEngine: (args['ab-local-engine-only'] as bool?) ?? false,
     );
   } else {
     await runTasks(taskNames,
@@ -142,6 +143,7 @@
   required String? deviceId,
   required String resultsFile,
   required String taskName,
+  bool onlyLocalEngine = false,
 }) async {
   print('$taskName A/B test. Will run $runsPerTest times.');
 
@@ -155,23 +157,27 @@
   for (int i = 1; i <= runsPerTest; i++) {
     section('Run #$i');
 
-    print('Running with the default engine (A)');
-    final TaskResult defaultEngineResult = await runTask(
-      taskName,
-      silent: silent,
-      deviceId: deviceId,
-    );
+    if (onlyLocalEngine) {
+      print('Skipping default engine (A)');
+    } else {
+      print('Running with the default engine (A)');
+      final TaskResult defaultEngineResult = await runTask(
+        taskName,
+        silent: silent,
+        deviceId: deviceId,
+      );
 
-    print('Default engine result:');
-    print(const JsonEncoder.withIndent('  ').convert(defaultEngineResult));
+      print('Default engine result:');
+      print(const JsonEncoder.withIndent('  ').convert(defaultEngineResult));
 
-    if (!defaultEngineResult.succeeded) {
-      stderr.writeln('Task failed on the default engine.');
-      exit(1);
+      if (!defaultEngineResult.succeeded) {
+        stderr.writeln('Task failed on the default engine.');
+        exit(1);
+      }
+
+      abTest.addAResult(defaultEngineResult);
     }
 
-    abTest.addAResult(defaultEngineResult);
-
     print('Running with the local engine (B)');
     final TaskResult localEngineResult = await runTask(
       taskName,
@@ -274,6 +280,11 @@
             'number if the name already exists.',
     )
     ..addFlag(
+      'ab-local-engine-only',
+      help: 'When running the A/B aggregator, do not run benchmarks with the default engine (A), only the local engine (B).\n'
+            'Shows the averages and noise report for the local engine without comparison to anything else.',
+    )
+    ..addFlag(
       'exit',
       help: 'Exit on the first test failure. Currently flakes are intentionally (though '
             'incorrectly) not considered to be failures.',