app_dart/lib/src/service/luci.dart - mirrors/cocoon - Git at Google

 // Copyright 2019 The Flutter Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 import 'dart:async';
 import 'dart:math';

 import 'package:appengine/appengine.dart';
 import 'package:json_annotation/json_annotation.dart';
 import 'package:meta/meta.dart';
 import 'package:retry/retry.dart';

 import '../model/appengine/task.dart';
 import '../model/ci_yaml/target.dart';
 import '../model/luci/buildbucket.dart';
 import '../request_handling/api_request_handler.dart';

 import 'buildbucket.dart';
 import 'config.dart';

 part 'luci.g.dart';

 const int _maxResults = 40;

 /// The batch size used to query buildbucket service.
 const int _buildersBatchSize = 50;

 const Map<Status, String> luciStatusToTaskStatus = <Status, String>{
   Status.unspecified: Task.statusInProgress,
   Status.scheduled: Task.statusInProgress,
   Status.started: Task.statusInProgress,
   Status.canceled: Task.statusSkipped,
   Status.success: Task.statusSucceeded,
   Status.failure: Task.statusFailed,
   Status.infraFailure: Task.statusInfraFailure,
 };

 typedef LuciServiceProvider = LuciService Function(ApiRequestHandler<dynamic> handler);

 /// Service class for interacting with LUCI.
 @immutable
 class LuciService {
   /// Creates a new [LuciService].
   ///
   /// The [buildBucketClient], [config], and [clientContext] arguments must not be null.
   const LuciService({
     required this.buildBucketClient,
     required this.config,
     required this.clientContext,
   });

   /// Client for making buildbucket requests to.
   final BuildBucketClient buildBucketClient;

   /// The Cocoon configuration. Guaranteed to be non-null.
   final Config config;

   /// The AppEngine context to use for requests. Guaranteed to be non-null.
   final ClientContext clientContext;

   /// Gets the list of recent LUCI tasks, broken out by the [BranchLuciBuilder]
   ///  that owns them.
   ///
   /// The list of known LUCI builders is specified in [LuciBuilder.all].
   Future<Map<BranchLuciBuilder, Map<String, List<LuciTask>>>> getBranchRecentTasks({
     required List<LuciBuilder> builders,
     bool requireTaskName = false,
   }) async {
     final List<Build> builds = await getBuildsForBuilderList(builders);

     final Map<BranchLuciBuilder, Map<String, List<LuciTask>>> results =
         <BranchLuciBuilder, Map<String, List<LuciTask>>>{};
     for (Build build in builds) {
       final String commit = build.input?.gitilesCommit?.hash ?? 'unknown';
       final String ref = build.input?.gitilesCommit?.ref ?? 'unknown';
       final LuciBuilder builder = builders.where((LuciBuilder builder) {
         return builder.name == build.builderId.builder;
       }).first;
       final String branch = ref == 'unknown' ? 'unknown' : ref.split('/')[2];
       final BranchLuciBuilder branchLuciBuilder = BranchLuciBuilder(
         luciBuilder: builder,
         branch: branch,
       );
       results[branchLuciBuilder] ??= <String, List<LuciTask>>{};
       results[branchLuciBuilder]![commit] ??= <LuciTask>[];
       results[branchLuciBuilder]![commit]!.add(LuciTask(
         commitSha: commit,
         ref: ref,
         status: luciStatusToTaskStatus[build.status!]!,
         buildNumber: build.number!,
         builderName: build.builderId.builder!,
         summaryMarkdown: build.summaryMarkdown,
       ));
     }
     return results;
   }

   /// Divides a large builder list `builders` to a list of smaller builder lists.
   @visibleForTesting
   List<List<LuciBuilder>> getPartialBuildersList(List<LuciBuilder> builders, int builderBatchSize) {
     final List<List<LuciBuilder>> partialBuildersList = <List<LuciBuilder>>[];
     for (int j = 0; j < builders.length; j += builderBatchSize) {
       partialBuildersList.add(builders.sublist(j, min(j + builderBatchSize, builders.length)));
     }
     return partialBuildersList;
   }

   /// Gets builds associated with a list of [builders] in batches with
   /// retries for [repo] including the task name or not.
   Future<List<Build>> getBuildsForBuilderList(
     List<LuciBuilder> builders, {
     bool requireTaskName = false,
   }) async {
     final List<Build> builds = <Build>[];
     // Request builders data in batches of 50 to prevent failures in the grpc service.
     const RetryOptions r = RetryOptions(maxAttempts: 3);
     final List<List<LuciBuilder>> partialBuildersList = getPartialBuildersList(builders, _buildersBatchSize);
     for (List<LuciBuilder> partialBuilders in partialBuildersList) {
       await r.retry(
         () async {
           final Iterable<Build> partialBuilds = await getBuilds(requireTaskName, partialBuilders);
           builds.addAll(partialBuilds);
         },
         retryIf: (Exception e) => e is BuildBucketException,
       );
       // Wait in between requests to prevent rate limiting.
       final Random random = Random();
       await Future<dynamic>.delayed(Duration(seconds: random.nextInt(10)));
     }
     return builds;
   }

   /// Gets the list of recent LUCI tasks, broken out by the [LuciBuilder] that
   /// owns them.
   ///
   /// The list of known LUCI builders is specified in [LuciBuilder.all].
   Future<Map<LuciBuilder, List<LuciTask>>> getRecentTasks({
     required List<LuciBuilder> builders,
     bool requireTaskName = false,
   }) async {
     final List<Build> builds = await getBuildsForBuilderList(builders);

     final Map<LuciBuilder, List<LuciTask>> results = <LuciBuilder, List<LuciTask>>{};
     for (Build build in builds) {
       final String commit = build.input?.gitilesCommit?.hash ?? 'unknown';
       final String ref = build.input?.gitilesCommit?.ref ?? 'unknown';
       final LuciBuilder builder = builders.where((LuciBuilder builder) {
         return builder.name == build.builderId.builder;
       }).first;
       results[builder] ??= <LuciTask>[];
       results[builder]!.add(LuciTask(
         commitSha: commit,
         ref: ref,
         status: luciStatusToTaskStatus[build.status!]!,
         buildNumber: build.number!,
         builderName: build.builderId.builder!,
         summaryMarkdown: build.summaryMarkdown,
       ));
     }
     return results;
   }

   /// Gets list of [build] for [repo] and available Luci [builders]
   /// predefined in cocoon config.
   ///
   /// Latest builds of each builder will be returned from new to old.
   Future<Iterable<Build>> getBuilds(bool requireTaskName, List<LuciBuilder> builders) async {
     bool includeBuilder(LuciBuilder builder) {
       if (requireTaskName && builder.taskName == null) {
         return false;
       }
       return true;
     }

     final List<Request> searchRequests = builders.where(includeBuilder).map<Request>((LuciBuilder builder) {
       return Request(
         searchBuilds: SearchBuildsRequest(
           pageSize: _maxResults,
           predicate: BuildPredicate(
             tags: <String, List<String>>{
               'scheduler_job_id': <String>['flutter/${builder.name}']
             },
           ),
           fields:
               'builds.*.id,builds.*.input,builds.*.builder,builds.*.number,builds.*.status,builds.*.summaryMarkdown',
         ),
       );
     }).toList();
     final BatchRequest batchRequest = BatchRequest(requests: searchRequests);
     final BatchResponse batchResponse = await buildBucketClient.batch(batchRequest);
     final Iterable<Build> builds = batchResponse.responses!
         .map<SearchBuildsResponse?>((Response response) => response.searchBuilds)
         .where((SearchBuildsResponse? response) => response?.builds != null)
         .expand<Build>((SearchBuildsResponse? response) => response?.builds ?? <Build>[]);

     return builds;
   }
 }

 @immutable
 class BranchLuciBuilder {
   const BranchLuciBuilder({
     this.luciBuilder,
     this.branch,
   });

   final String? branch;
   final LuciBuilder? luciBuilder;

   @override
   int get hashCode => '${luciBuilder.toString()},$branch'.hashCode;

   @override
   bool operator ==(Object other) =>
       other is BranchLuciBuilder &&
       other.luciBuilder!.name == luciBuilder!.name &&
       other.luciBuilder!.taskName == luciBuilder!.taskName &&
       other.luciBuilder!.repo == luciBuilder!.repo &&
       other.luciBuilder!.flaky == luciBuilder!.flaky;
 }

 @immutable
 @JsonSerializable()
 class LuciBuilder {
   const LuciBuilder({
     required this.name,
     required this.repo,
     required this.flaky,
     this.enabled,
     this.runIf,
     this.taskName,
   });

   /// Create a new [LuciBuilder] from a [Target].
   factory LuciBuilder.fromTarget(Target target) {
     return LuciBuilder(
       name: target.value.name,
       repo: target.slug.name,
       runIf: target.value.runIf,
       taskName: target.value.name,
       flaky: target.value.bringup,
     );
   }

   /// The name of this builder.
   @JsonKey(required: true, disallowNullValue: true)
   final String name;

   /// The name of the repository for which this builder runs.
   @JsonKey(required: true, disallowNullValue: true)
   final String? repo;

   /// Flag the result of this builder as blocker or not.
   @JsonKey()
   final bool? flaky;

   /// Flag if this builder is enabled or not.
   @JsonKey(name: 'enabled')
   final bool? enabled;

   /// Globs to filter changed files to trigger builders.
   @JsonKey(name: 'run_if')
   final List<String>? runIf;

   /// The name of the devicelab task associated with this builder.
   @JsonKey(name: 'task_name')
   final String? taskName;

   /// Serializes this object to a JSON primitive.
   Map<String, dynamic> toJson() => _$LuciBuilderToJson(this);

   @override
   bool operator ==(Object other) {
     if (other is! LuciBuilder) {
       return false;
     }
     return name == other.name && repo == other.repo;
   }

   @override
   String toString() {
     return '''LuciBuilder(
       name: $name
       repo: $repo
       flaky: $flaky
       enabled: $enabled
       runIf: $runIf
       taskName: $taskName
     )''';
   }

   @override
   int get hashCode => name.hashCode ^ repo.hashCode;
 }

 @immutable
 class LuciTask {
   const LuciTask({
     required this.commitSha,
     required this.ref,
     required this.status,
     required this.buildNumber,
     required this.builderName,
     this.summaryMarkdown,
   });

   /// The GitHub commit at which this task is being run.
   final String commitSha;

   // The GitHub ref at which this task is being run.
   final String ref;

   /// The status of this task. See the [Task] class for supported values.
   final String status;

   /// The build number of this task.
   final int buildNumber;

   /// The builder name of this task.
   final String builderName;

   /// The builder name of this task.
   final String? summaryMarkdown;
 }
	// Copyright 2019 The Flutter Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	import 'dart:async';
	import 'dart:math';

	import 'package:appengine/appengine.dart';
	import 'package:json_annotation/json_annotation.dart';
	import 'package:meta/meta.dart';
	import 'package:retry/retry.dart';

	import '../model/appengine/task.dart';
	import '../model/ci_yaml/target.dart';
	import '../model/luci/buildbucket.dart';
	import '../request_handling/api_request_handler.dart';

	import 'buildbucket.dart';
	import 'config.dart';

	part 'luci.g.dart';

	const int _maxResults = 40;

	/// The batch size used to query buildbucket service.
	const int _buildersBatchSize = 50;

	const Map<Status, String> luciStatusToTaskStatus = <Status, String>{
	Status.unspecified: Task.statusInProgress,
	Status.scheduled: Task.statusInProgress,
	Status.started: Task.statusInProgress,
	Status.canceled: Task.statusSkipped,
	Status.success: Task.statusSucceeded,
	Status.failure: Task.statusFailed,
	Status.infraFailure: Task.statusInfraFailure,
	};

	typedef LuciServiceProvider = LuciService Function(ApiRequestHandler<dynamic> handler);

	/// Service class for interacting with LUCI.
	@immutable
	class LuciService {
	/// Creates a new [LuciService].
	///
	/// The [buildBucketClient], [config], and [clientContext] arguments must not be null.
	const LuciService({
	required this.buildBucketClient,
	required this.config,
	required this.clientContext,
	});

	/// Client for making buildbucket requests to.
	final BuildBucketClient buildBucketClient;

	/// The Cocoon configuration. Guaranteed to be non-null.
	final Config config;

	/// The AppEngine context to use for requests. Guaranteed to be non-null.
	final ClientContext clientContext;

	/// Gets the list of recent LUCI tasks, broken out by the [BranchLuciBuilder]
	/// that owns them.
	///
	/// The list of known LUCI builders is specified in [LuciBuilder.all].
	Future<Map<BranchLuciBuilder, Map<String, List<LuciTask>>>> getBranchRecentTasks({
	required List<LuciBuilder> builders,
	bool requireTaskName = false,
	}) async {
	final List<Build> builds = await getBuildsForBuilderList(builders);

	final Map<BranchLuciBuilder, Map<String, List<LuciTask>>> results =
	<BranchLuciBuilder, Map<String, List<LuciTask>>>{};
	for (Build build in builds) {
	final String commit = build.input?.gitilesCommit?.hash ?? 'unknown';
	final String ref = build.input?.gitilesCommit?.ref ?? 'unknown';
	final LuciBuilder builder = builders.where((LuciBuilder builder) {
	return builder.name == build.builderId.builder;
	}).first;
	final String branch = ref == 'unknown' ? 'unknown' : ref.split('/')[2];
	final BranchLuciBuilder branchLuciBuilder = BranchLuciBuilder(
	luciBuilder: builder,
	branch: branch,
	);
	results[branchLuciBuilder] ??= <String, List<LuciTask>>{};
	results[branchLuciBuilder]![commit] ??= <LuciTask>[];
	results[branchLuciBuilder]![commit]!.add(LuciTask(
	commitSha: commit,
	ref: ref,
	status: luciStatusToTaskStatus[build.status!]!,
	buildNumber: build.number!,
	builderName: build.builderId.builder!,
	summaryMarkdown: build.summaryMarkdown,
	));
	}
	return results;
	}

	/// Divides a large builder list `builders` to a list of smaller builder lists.
	@visibleForTesting
	List<List<LuciBuilder>> getPartialBuildersList(List<LuciBuilder> builders, int builderBatchSize) {
	final List<List<LuciBuilder>> partialBuildersList = <List<LuciBuilder>>[];
	for (int j = 0; j < builders.length; j += builderBatchSize) {
	partialBuildersList.add(builders.sublist(j, min(j + builderBatchSize, builders.length)));
	}
	return partialBuildersList;
	}

	/// Gets builds associated with a list of [builders] in batches with
	/// retries for [repo] including the task name or not.
	Future<List<Build>> getBuildsForBuilderList(
	List<LuciBuilder> builders, {
	bool requireTaskName = false,
	}) async {
	final List<Build> builds = <Build>[];
	// Request builders data in batches of 50 to prevent failures in the grpc service.
	const RetryOptions r = RetryOptions(maxAttempts: 3);
	final List<List<LuciBuilder>> partialBuildersList = getPartialBuildersList(builders, _buildersBatchSize);
	for (List<LuciBuilder> partialBuilders in partialBuildersList) {
	await r.retry(
	() async {
	final Iterable<Build> partialBuilds = await getBuilds(requireTaskName, partialBuilders);
	builds.addAll(partialBuilds);
	},
	retryIf: (Exception e) => e is BuildBucketException,
	);
	// Wait in between requests to prevent rate limiting.
	final Random random = Random();
	await Future<dynamic>.delayed(Duration(seconds: random.nextInt(10)));
	}
	return builds;
	}

	/// Gets the list of recent LUCI tasks, broken out by the [LuciBuilder] that
	/// owns them.
	///
	/// The list of known LUCI builders is specified in [LuciBuilder.all].
	Future<Map<LuciBuilder, List<LuciTask>>> getRecentTasks({
	required List<LuciBuilder> builders,
	bool requireTaskName = false,
	}) async {
	final List<Build> builds = await getBuildsForBuilderList(builders);

	final Map<LuciBuilder, List<LuciTask>> results = <LuciBuilder, List<LuciTask>>{};
	for (Build build in builds) {
	final String commit = build.input?.gitilesCommit?.hash ?? 'unknown';
	final String ref = build.input?.gitilesCommit?.ref ?? 'unknown';
	final LuciBuilder builder = builders.where((LuciBuilder builder) {
	return builder.name == build.builderId.builder;
	}).first;
	results[builder] ??= <LuciTask>[];
	results[builder]!.add(LuciTask(
	commitSha: commit,
	ref: ref,
	status: luciStatusToTaskStatus[build.status!]!,
	buildNumber: build.number!,
	builderName: build.builderId.builder!,
	summaryMarkdown: build.summaryMarkdown,
	));
	}
	return results;
	}

	/// Gets list of [build] for [repo] and available Luci [builders]
	/// predefined in cocoon config.
	///
	/// Latest builds of each builder will be returned from new to old.
	Future<Iterable<Build>> getBuilds(bool requireTaskName, List<LuciBuilder> builders) async {
	bool includeBuilder(LuciBuilder builder) {
	if (requireTaskName && builder.taskName == null) {
	return false;
	}
	return true;
	}

	final List<Request> searchRequests = builders.where(includeBuilder).map<Request>((LuciBuilder builder) {
	return Request(
	searchBuilds: SearchBuildsRequest(
	pageSize: _maxResults,
	predicate: BuildPredicate(
	tags: <String, List<String>>{
	'scheduler_job_id': <String>['flutter/${builder.name}']
	},
	),
	fields:
	'builds..id,builds..input,builds..builder,builds..number,builds..status,builds..summaryMarkdown',
	),
	);
	}).toList();
	final BatchRequest batchRequest = BatchRequest(requests: searchRequests);
	final BatchResponse batchResponse = await buildBucketClient.batch(batchRequest);
	final Iterable<Build> builds = batchResponse.responses!
	.map<SearchBuildsResponse?>((Response response) => response.searchBuilds)
	.where((SearchBuildsResponse? response) => response?.builds != null)
	.expand<Build>((SearchBuildsResponse? response) => response?.builds ?? <Build>[]);

	return builds;
	}
	}

	@immutable
	class BranchLuciBuilder {
	const BranchLuciBuilder({
	this.luciBuilder,
	this.branch,
	});

	final String? branch;
	final LuciBuilder? luciBuilder;

	@override
	int get hashCode => '${luciBuilder.toString()},$branch'.hashCode;

	@override
	bool operator ==(Object other) =>
	other is BranchLuciBuilder &&
	other.luciBuilder!.name == luciBuilder!.name &&
	other.luciBuilder!.taskName == luciBuilder!.taskName &&
	other.luciBuilder!.repo == luciBuilder!.repo &&
	other.luciBuilder!.flaky == luciBuilder!.flaky;
	}

	@immutable
	@JsonSerializable()
	class LuciBuilder {
	const LuciBuilder({
	required this.name,
	required this.repo,
	required this.flaky,
	this.enabled,
	this.runIf,
	this.taskName,
	});

	/// Create a new [LuciBuilder] from a [Target].
	factory LuciBuilder.fromTarget(Target target) {
	return LuciBuilder(
	name: target.value.name,
	repo: target.slug.name,
	runIf: target.value.runIf,
	taskName: target.value.name,
	flaky: target.value.bringup,
	);
	}

	/// The name of this builder.
	@JsonKey(required: true, disallowNullValue: true)
	final String name;

	/// The name of the repository for which this builder runs.
	@JsonKey(required: true, disallowNullValue: true)
	final String? repo;

	/// Flag the result of this builder as blocker or not.
	@JsonKey()
	final bool? flaky;

	/// Flag if this builder is enabled or not.
	@JsonKey(name: 'enabled')
	final bool? enabled;

	/// Globs to filter changed files to trigger builders.
	@JsonKey(name: 'run_if')
	final List<String>? runIf;

	/// The name of the devicelab task associated with this builder.
	@JsonKey(name: 'task_name')
	final String? taskName;

	/// Serializes this object to a JSON primitive.
	Map<String, dynamic> toJson() => _$LuciBuilderToJson(this);

	@override
	bool operator ==(Object other) {
	if (other is! LuciBuilder) {
	return false;
	}
	return name == other.name && repo == other.repo;
	}

	@override
	String toString() {
	return '''LuciBuilder(
	name: $name
	repo: $repo
	flaky: $flaky
	enabled: $enabled
	runIf: $runIf
	taskName: $taskName
	)''';
	}

	@override
	int get hashCode => name.hashCode ^ repo.hashCode;
	}

	@immutable
	class LuciTask {
	const LuciTask({
	required this.commitSha,
	required this.ref,
	required this.status,
	required this.buildNumber,
	required this.builderName,
	this.summaryMarkdown,
	});

	/// The GitHub commit at which this task is being run.
	final String commitSha;

	// The GitHub ref at which this task is being run.
	final String ref;

	/// The status of this task. See the [Task] class for supported values.
	final String status;

	/// The build number of this task.
	final int buildNumber;

	/// The builder name of this task.
	final String builderName;

	/// The builder name of this task.
	final String? summaryMarkdown;
	}