add backfillerTargetLimit (#2529)
Enforce `backfillerTargetLimit` when backfilling
diff --git a/app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart b/app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart
index 932b823..2a9ac1a 100644
--- a/app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart
+++ b/app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart
@@ -62,7 +62,7 @@
}
// Check if should be scheduled (there is no yellow runs). Run the most recent gray.
- final List<Tuple<Target, FullTask, int>> backfill = <Tuple<Target, FullTask, int>>[];
+ List<Tuple<Target, FullTask, int>> backfill = <Tuple<Target, FullTask, int>>[];
for (List<FullTask> taskColumn in taskMap.values) {
final FullTask task = taskColumn.first;
final CiYaml ciYaml = await scheduler.getCiYaml(task.commit);
@@ -77,6 +77,11 @@
}
}
+ // Limits the number of targets to be backfilled in each cycle.
+ if (backfill.length > config.backfillerTargetLimit) {
+ backfill = backfill.sublist(0, config.backfillerTargetLimit);
+ }
+
log.fine('Backfilling ${backfill.length} builds');
log.fine(backfill.map<String>((Tuple<Target, FullTask, int> tuple) => tuple.first.value.name));
diff --git a/app_dart/lib/src/service/config.dart b/app_dart/lib/src/service/config.dart
index 749be04..9cd36c0 100644
--- a/app_dart/lib/src/service/config.dart
+++ b/app_dart/lib/src/service/config.dart
@@ -151,6 +151,9 @@
/// Batch size of builds to schedule in each swarming request.
int get batchSize => 5;
+ /// Upper limit of targets to be backfilled in API call.
+ int get backfillerTargetLimit => 50;
+
/// Max retries when scheduling builds.
static const RetryOptions schedulerRetry = RetryOptions(maxAttempts: 3);
diff --git a/app_dart/test/request_handlers/scheduler/batch_backfiller_test.dart b/app_dart/test/request_handlers/scheduler/batch_backfiller_test.dart
index d4482b2..0269260 100644
--- a/app_dart/test/request_handlers/scheduler/batch_backfiller_test.dart
+++ b/app_dart/test/request_handlers/scheduler/batch_backfiller_test.dart
@@ -35,7 +35,7 @@
group('BatchBackfiller', () {
setUp(() async {
db = FakeDatastoreDB()..addOnQuery<Commit>((Iterable<Commit> results) => commits);
- final Config config = FakeConfig(dbValue: db);
+ final Config config = FakeConfig(dbValue: db, backfillerTargetLimitValue: 2);
pubsub = FakePubSub();
mockGithubChecksUtil = MockGithubChecksUtil();
when(
@@ -185,5 +185,19 @@
await tester.get(handler);
expect(pubsub.messages.length, 2);
});
+
+ test('backfills limited targets when number of available targets exceeds backfillerTargetLimit ', () async {
+ final List<Task> scheduleA = <Task>[
+ // Linux_android A
+ generateTask(1, name: 'Linux_android A', status: Task.statusNew),
+ // Linux_android B
+ generateTask(1, name: 'Linux_android B', status: Task.statusNew),
+ // Linux_android C
+ generateTask(1, name: 'Linux_android C', status: Task.statusNew),
+ ];
+ db.addOnQuery<Task>((Iterable<Task> results) => scheduleA);
+ await tester.get(handler);
+ expect(pubsub.messages.length, 2);
+ });
});
}
diff --git a/app_dart/test/src/datastore/fake_config.dart b/app_dart/test/src/datastore/fake_config.dart
index cc8df3d..8440a2e 100644
--- a/app_dart/test/src/datastore/fake_config.dart
+++ b/app_dart/test/src/datastore/fake_config.dart
@@ -57,6 +57,7 @@
this.supportedBranchesValue,
this.supportedReposValue,
this.batchSizeValue,
+ this.backfillerTargetLimitValue,
this.githubRequestDelayValue,
FakeDatastoreDB? dbValue,
}) : dbValue = dbValue ?? FakeDatastoreDB();
@@ -87,6 +88,7 @@
String? waitingForTreeToGoGreenLabelNameValue;
Set<String>? rollerAccountsValue;
int? maxRecordsValue;
+ int? backfillerTargetLimitValue;
String? flutterGoldPendingValue;
String? flutterGoldSuccessValue;
String? flutterGoldChangesValue;
@@ -139,6 +141,9 @@
int get schedulingShardSize => 5;
@override
+ int get backfillerTargetLimit => backfillerTargetLimitValue ?? 50;
+
+ @override
int get batchSize => batchSizeValue ?? 5;
@override
diff --git a/app_dart/test/src/service/fake_scheduler.dart b/app_dart/test/src/service/fake_scheduler.dart
index d01d85e..ce9694f 100644
--- a/app_dart/test/src/service/fake_scheduler.dart
+++ b/app_dart/test/src/service/fake_scheduler.dart
@@ -152,6 +152,9 @@
pb.Target(
name: 'Linux_android B',
),
+ pb.Target(
+ name: 'Linux_android C',
+ ),
],
),
);
