Autosubmit not working plugins (#1949)
diff --git a/app_dart/lib/src/service/github_checks_service.dart b/app_dart/lib/src/service/github_checks_service.dart
index 1574168..a544aee 100644
--- a/app_dart/lib/src/service/github_checks_service.dart
+++ b/app_dart/lib/src/service/github_checks_service.dart
@@ -147,7 +147,7 @@
}
}
- /// Transforms a [ush_message.Status] to a [github.CheckRunStatus].
+ /// Transforms a [push_message.Status] to a [github.CheckRunStatus].
/// Relevant APIs:
/// https://developer.github.com/v3/checks/runs/#check-runs
github.CheckRunStatus statusForResult(push_message.Status? status) {
diff --git a/auto_submit/lib/validations/ci_successful.dart b/auto_submit/lib/validations/ci_successful.dart
index 086f20e..c66ced4 100644
--- a/auto_submit/lib/validations/ci_successful.dart
+++ b/auto_submit/lib/validations/ci_successful.dart
@@ -12,6 +12,13 @@
/// Validates all the CI build/tests ran and were successful.
class CiSuccessful extends Validation {
+ /// The status checks that are not related to changes in this PR.
+ static const Set<String> notInAuthorsControl = <String>{
+ 'luci-flutter', // flutter repo
+ 'luci-engine', // engine repo
+ 'submit-queue', // plugins repo
+ };
+
CiSuccessful({
required Config config,
}) : super(config: config);
@@ -25,71 +32,37 @@
final PullRequest pullRequest = result.repository!.pullRequest!;
Set<FailureDetail> failures = <FailureDetail>{};
- // The status checks that are not related to changes in this PR.
- const Set<String> notInAuthorsControl = <String>{
- 'luci-flutter', // flutter repo
- 'luci-engine', // engine repo
- 'submit-queue', // plugins repo
- };
List<ContextNode> statuses = <ContextNode>[];
Commit commit = pullRequest.commits!.nodes!.single.commit!;
+
// Recently most of the repositories have migrated away of using the status
// APIs and for those repos commit.status is null.
if (commit.status != null && commit.status!.contexts!.isNotEmpty) {
statuses.addAll(commit.status!.contexts!);
}
- // Ensure repos with tree statuses have it set
- if (Config.reposWithTreeStatus.contains(slug)) {
- bool treeStatusExists = false;
- final String treeStatusName = 'luci-${slug.name}';
- // Scan list of statuses to see if the tree status exists (this list is expected to be <5 items)
- for (ContextNode status in statuses) {
- if (status.context == treeStatusName) {
- treeStatusExists = true;
- }
- }
+ /// Validate tree statuses are set.
+ validateTreeStatusIsSet(slug, statuses, failures);
- if (!treeStatusExists) {
- failures.add(FailureDetail('tree status $treeStatusName', 'https://flutter-dashboard.appspot.com/#/build'));
- }
- }
// List of labels associated with the pull request.
final List<String> labelNames = (messagePullRequest.labels as List<github.IssueLabel>)
.map<String>((github.IssueLabel labelMap) => labelMap.name)
.toList();
- final String overrideTreeStatusLabel = config.overrideTreeStatusLabel;
- log.info('Validating name: ${slug.name}, status: $statuses');
- for (ContextNode status in statuses) {
- final String? name = status.context;
- if (status.state != STATUS_SUCCESS) {
- if (notInAuthorsControl.contains(name) && labelNames.contains(overrideTreeStatusLabel)) {
- continue;
- }
- allSuccess = false;
- if (status.state == STATUS_FAILURE && !notInAuthorsControl.contains(name)) {
- failures.add(FailureDetail(name!, status.targetUrl!));
- }
- }
- }
+
+ /// Validate if all statuses have been successful.
+ allSuccess = validateStatuses(slug, labelNames, statuses, failures, allSuccess);
+
final GithubService gitHubService = await config.createGithubService(slug);
final String? sha = commit.oid;
+
List<github.CheckRun> checkRuns = <github.CheckRun>[];
if (messagePullRequest.head != null && sha != null) {
checkRuns.addAll(await gitHubService.getCheckRuns(slug, sha));
}
- log.info('Validating name: ${slug.name}, checks: $checkRuns');
- for (github.CheckRun checkRun in checkRuns) {
- final String? name = checkRun.name;
- if (checkRun.conclusion == github.CheckRunConclusion.success ||
- (checkRun.status == github.CheckRunStatus.completed &&
- checkRun.conclusion == github.CheckRunConclusion.neutral)) {
- continue;
- } else if (checkRun.status == github.CheckRunStatus.completed) {
- failures.add(FailureDetail(name!, checkRun.detailsUrl as String));
- }
- allSuccess = false;
- }
+
+ /// Validate if all checkRuns have succeeded.
+ allSuccess = validateCheckRuns(slug, checkRuns, failures, allSuccess);
+
if (!allSuccess && failures.isEmpty) {
return ValidationResult(allSuccess, Action.IGNORE_TEMPORARILY, '');
}
@@ -103,4 +76,80 @@
Action action = labelNames.contains(config.overrideTreeStatusLabel) ? Action.IGNORE_FAILURE : Action.REMOVE_LABEL;
return ValidationResult(allSuccess, action, buffer.toString());
}
+
+ /// Validate that the tree status exists for all statuses in the supplied list.
+ ///
+ /// If a failure is found it is added to the set of overall failures.
+ void validateTreeStatusIsSet(github.RepositorySlug slug, List<ContextNode> statuses, Set<FailureDetail> failures) {
+ if (Config.reposWithTreeStatus.contains(slug)) {
+ bool treeStatusExists = false;
+ final String treeStatusName = 'luci-${slug.name}';
+
+ /// Scan list of statuses to see if the tree status exists (this list is expected to be <5 items)
+ for (ContextNode status in statuses) {
+ if (status.context == treeStatusName) {
+ // Does only one tree status need to be set for the condition?
+ treeStatusExists = true;
+ }
+ }
+
+ if (!treeStatusExists) {
+ failures.add(FailureDetail('tree status $treeStatusName', 'https://flutter-dashboard.appspot.com/#/build'));
+ }
+ }
+ }
+
+ /// Validate the ci build test run statuses to see which have succeeded and
+ /// which have failed.
+ ///
+ /// Failures will be added the set of overall failures.
+ /// Returns allSuccess unmodified if there were no failures, false otherwise.
+ bool validateStatuses(github.RepositorySlug slug, List<String> labelNames, List<ContextNode> statuses,
+ Set<FailureDetail> failures, bool allSuccess) {
+ final String overrideTreeStatusLabel = config.overrideTreeStatusLabel;
+
+ log.info('Validating name: ${slug.name}, status: $statuses');
+ for (ContextNode status in statuses) {
+ // How can name be null but presumed to not be null below when added to failure?
+ final String? name = status.context;
+
+ if (status.state != STATUS_SUCCESS) {
+ if (notInAuthorsControl.contains(name) && labelNames.contains(overrideTreeStatusLabel)) {
+ continue;
+ }
+ allSuccess = false;
+ if (status.state == STATUS_FAILURE && !notInAuthorsControl.contains(name)) {
+ failures.add(FailureDetail(name!, status.targetUrl!));
+ }
+ }
+ }
+
+ return allSuccess;
+ }
+
+ /// Validate the checkRuns to see if all have completed successfully or not.
+ ///
+ /// Failures will be added the set of overall failures.
+ /// Returns allSuccess unmodified if there were no failures, false otherwise.
+ bool validateCheckRuns(
+ github.RepositorySlug slug, List<github.CheckRun> checkRuns, Set<FailureDetail> failures, bool allSuccess) {
+ log.info('Validating name: ${slug.name}, checks: $checkRuns');
+ for (github.CheckRun checkRun in checkRuns) {
+ final String? name = checkRun.name;
+
+ if (checkRun.conclusion == github.CheckRunConclusion.skipped ||
+ checkRun.conclusion == github.CheckRunConclusion.success ||
+ (checkRun.status == github.CheckRunStatus.completed &&
+ checkRun.conclusion == github.CheckRunConclusion.neutral)) {
+ // checkrun has passed.
+ continue;
+ } else if (checkRun.status == github.CheckRunStatus.completed) {
+ // checkrun has failed.
+ failures.add(FailureDetail(name!, checkRun.detailsUrl as String));
+ }
+ allSuccess = false;
+ }
+
+ return allSuccess;
+ }
}
diff --git a/auto_submit/test/requests/github_webhook_test_data.dart b/auto_submit/test/requests/github_webhook_test_data.dart
index fd479ba..684e92e 100644
--- a/auto_submit/test/requests/github_webhook_test_data.dart
+++ b/auto_submit/test/requests/github_webhook_test_data.dart
@@ -151,7 +151,7 @@
}
]''';
-String unApprovedReviewsMock = '''[
+const String unApprovedReviewsMock = '''[
{
"id": 81,
"user": {
@@ -221,7 +221,7 @@
]
}''';
-String inProgressCheckRunsMock = '''{
+const String inProgressCheckRunsMock = '''{
"total_count": 1,
"check_runs": [
{
@@ -240,6 +240,134 @@
]
}''';
+const String skippedCheckRunsMock = '''{
+ "total_count": 1,
+ "check_runs": [
+ {
+ "id": 6,
+ "head_sha": "be6ff099a4ee56e152a5fa2f37edd10f79d1269a",
+ "external_id": "",
+ "details_url": "https://example.com",
+ "status": "in_progress",
+ "conclusion": "skipped",
+ "started_at": "2018-05-04T01:14:52Z",
+ "name": "inprogress_checkrun",
+ "check_suite": {
+ "id": 5
+ }
+ }
+ ]
+}''';
+
+const String multipleCheckRunsMock = '''{
+ "total_count": 3,
+ "check_runs": [
+ {
+ "id": 1,
+ "head_sha": "be6ff099a4ee56e152a5fa2f37edd10f79d1269a",
+ "external_id": "",
+ "details_url": "https://example.com",
+ "status": "completed",
+ "conclusion": "success",
+ "started_at": "2018-05-04T01:14:52Z",
+ "name": "mighty_readme",
+ "check_suite": {
+ "id": 5
+ }
+ },
+ {
+ "id": 2,
+ "head_sha": "be6ff099a4ee56e152a5fa2f37edd10f79d1269a",
+ "external_id": "",
+ "details_url": "https://example.com",
+ "status": "completed",
+ "conclusion": "neutral",
+ "started_at": "2018-05-04T01:14:52Z",
+ "name": "neutral_checkrun",
+ "check_suite": {
+ "id": 5
+ }
+ },
+ {
+ "id": 6,
+ "head_sha": "be6ff099a4ee56e152a5fa2f37edd10f79d1269a",
+ "external_id": "",
+ "details_url": "https://example.com",
+ "status": "in_progress",
+ "conclusion": "skipped",
+ "started_at": "2018-05-04T01:14:52Z",
+ "name": "inprogress_checkrun",
+ "check_suite": {
+ "id": 5
+ }
+ }
+ ]
+}''';
+
+const String multipleCheckRunsWithFailureMock = '''{
+ "total_count": 3,
+ "check_runs": [
+ {
+ "id": 1,
+ "head_sha": "be6ff099a4ee56e152a5fa2f37edd10f79d1269a",
+ "external_id": "",
+ "details_url": "https://example.com",
+ "status": "completed",
+ "conclusion": "success",
+ "started_at": "2018-05-04T01:14:52Z",
+ "name": "mighty_readme",
+ "check_suite": {
+ "id": 5
+ }
+ },
+ {
+ "id": 2,
+ "head_sha": "be6ff099a4ee56e152a5fa2f37edd10f79d1269a",
+ "external_id": "",
+ "details_url": "https://example.com",
+ "status": "completed",
+ "conclusion": "failure",
+ "started_at": "2018-05-04T01:14:52Z",
+ "name": "failed_checkrun",
+ "check_suite": {
+ "id": 5
+ }
+ },
+ {
+ "id": 6,
+ "head_sha": "be6ff099a4ee56e152a5fa2f37edd10f79d1269a",
+ "external_id": "",
+ "details_url": "https://example.com",
+ "status": "in_progress",
+ "conclusion": "skipped",
+ "started_at": "2018-05-04T01:14:52Z",
+ "name": "inprogress_checkrun",
+ "check_suite": {
+ "id": 5
+ }
+ }
+ ]
+}''';
+
+const String inprogressAndNotFailedCheckRunMock = '''{
+ "total_count": 1,
+ "check_runs": [
+ {
+ "id": 6,
+ "head_sha": "be6ff099a4ee56e152a5fa2f37edd10f79d1269a",
+ "external_id": "",
+ "details_url": "https://example.com",
+ "status": "in_progress",
+ "conclusion": "neutral",
+ "started_at": "2018-05-04T01:14:52Z",
+ "name": "inprogress_checkrun",
+ "check_suite": {
+ "id": 5
+ }
+ }
+ ]
+}''';
+
const String emptyCheckRunsMock = '''{"check_runs": [{}]}''';
// repositoryStatusesMock is from the official Github API: https://developer.github.com/v3/repos/statuses/#list-statuses-for-a-specific-ref
@@ -257,34 +385,48 @@
]
}''';
-String failedAuthorsStatusesMock = '''{
- "state": "failure",
+const String repositoryStatusesNonLuciFlutterMock = '''{
+ "state": "success",
"statuses": [
{
- "state": "failure",
- "context": "luci-flutter",
- "target_url": "https://ci.example.com/1000/output"
+ "state": "success",
+ "context": "infra"
},
{
- "state": "failure",
- "context": "luci-engine",
- "target_url": "https://ci.example.com/2000/output"
+ "state": "success",
+ "context": "config"
}
]
}''';
-String failedNonAuthorsStatusesMock = '''{
+const String failedAuthorsStatusesMock = '''{
"state": "failure",
"statuses": [
{
"state": "failure",
"context": "luci-flutter",
- "target_url": "https://ci.example.com/1000/output"
+ "targetUrl": "https://ci.example.com/1000/output"
},
{
"state": "failure",
- "context": "flutter-cocoon",
- "target_url": "https://ci.example.com/2000/output"
+ "context": "luci-engine",
+ "targetUrl": "https://ci.example.com/2000/output"
+ }
+ ]
+}''';
+
+const String failedNonAuthorsStatusesMock = '''{
+ "state": "failure",
+ "statuses": [
+ {
+ "state": "failure",
+ "context": "flutter-engine",
+ "targetUrl": "https://ci.example.com/1000/output"
+ },
+ {
+ "state": "failure",
+ "context": "flutter-infra",
+ "targetUrl": "https://ci.example.com/2000/output"
}
]
}''';
diff --git a/auto_submit/test/validations/ci_successful_test.dart b/auto_submit/test/validations/ci_successful_test.dart
index ba19fde..fb8518b 100644
--- a/auto_submit/test/validations/ci_successful_test.dart
+++ b/auto_submit/test/validations/ci_successful_test.dart
@@ -2,45 +2,373 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
-import 'package:auto_submit/model/auto_submit_query_result.dart' hide PullRequest;
-import 'package:auto_submit/validations/ci_successful.dart';
-import 'package:auto_submit/validations/validation.dart';
-import 'package:github/github.dart';
-import 'package:test/test.dart';
+import 'dart:convert';
-import '../requests/github_webhook_test_data.dart';
-import '../src/service/fake_config.dart';
-import '../src/service/fake_github_service.dart';
+import 'ci_successful_test_data.dart';
+
+import 'package:github/github.dart' as github;
+import 'package:test/test.dart';
+import 'package:auto_submit/validations/ci_successful.dart';
+import 'package:auto_submit/model/auto_submit_query_result.dart';
+import 'package:auto_submit/validations/validation.dart';
+
import '../utilities/utils.dart';
import '../utilities/mocks.dart';
+import '../src/service/fake_config.dart';
+import '../src/service/fake_github_service.dart';
+import '../src/service/fake_graphql_client.dart';
+import '../requests/github_webhook_test_data.dart';
void main() {
- late FakeConfig config;
late CiSuccessful ciSuccessful;
- late FakeGithubService githubService;
- late RepositorySlug slug;
+ late FakeConfig config;
+ FakeGithubService githubService = FakeGithubService();
+ late FakeGraphQLClient githubGraphQLClient;
+ MockGitHub gitHub = MockGitHub();
+ late github.RepositorySlug slug;
+ late Set<FailureDetail> failures;
+ List<ContextNode> _getContextNodeListFromJson(final String repositoryStatuses) {
+ List<ContextNode> contextNodeList = [];
+
+ Map<String, dynamic> contextNodeMap = jsonDecode(repositoryStatuses) as Map<String, dynamic>;
+
+ dynamic statuses = contextNodeMap['statuses'];
+ for (Map<String, dynamic> map in statuses) {
+ contextNodeList.add(ContextNode.fromJson(map));
+ }
+
+ return contextNodeList;
+ }
+
+ void _convertContextNodeStatuses(List<ContextNode> contextNodeList) {
+ for (ContextNode contextNode in contextNodeList) {
+ contextNode.state = contextNode.state!.toUpperCase();
+ }
+ }
+
+ /// Setup objects needed across test groups.
setUp(() {
- githubService = FakeGithubService(client: MockGitHub());
- config = FakeConfig(githubService: githubService);
+ githubGraphQLClient = FakeGraphQLClient();
+ config = FakeConfig(githubService: githubService, githubGraphQLClient: githubGraphQLClient, githubClient: gitHub);
ciSuccessful = CiSuccessful(config: config);
- slug = RepositorySlug('flutter', 'cocoon');
+ slug = github.RepositorySlug('octocat', 'flutter');
+ failures = <FailureDetail>{};
});
- test('returns correct message when validation fails', () async {
- PullRequestHelper flutterRequest = PullRequestHelper(
- prNumber: 0,
- lastCommitHash: oid,
- reviews: <PullRequestReviewHelper>[],
- );
- githubService.checkRunsData = failedCheckRunsMock;
- final PullRequest pullRequest = generatePullRequest(prNumber: 0, repoName: slug.name);
- QueryResult queryResult = createQueryResult(flutterRequest);
+ group('validateCheckRuns', () {
+ test('ValidateCheckRuns no failures for skipped conclusion.', () {
+ githubService.checkRunsData = skippedCheckRunsMock;
+ final Future<List<github.CheckRun>> checkRunFuture = githubService.getCheckRuns(slug, 'ref');
+ bool allSuccess = true;
- final ValidationResult validationResult = await ciSuccessful.validate(queryResult, pullRequest);
+ checkRunFuture.then((checkRuns) {
+ expect(ciSuccessful.validateCheckRuns(slug, checkRuns, failures, allSuccess), isTrue);
+ expect(failures, isEmpty);
+ });
+ });
- expect(validationResult.result, false);
- expect(validationResult.message,
- '- The status or check suite [failed_checkrun](https://example.com) has failed. Please fix the issues identified (or deflake) before re-applying this label.\n');
+ test('ValidateCheckRuns no failures for successful conclusion.', () {
+ githubService.checkRunsData = checkRunsMock;
+ final Future<List<github.CheckRun>> checkRunFuture = githubService.getCheckRuns(slug, 'ref');
+ bool allSuccess = true;
+
+ checkRunFuture.then((checkRuns) {
+ expect(ciSuccessful.validateCheckRuns(slug, checkRuns, failures, allSuccess), isTrue);
+ expect(failures, isEmpty);
+ });
+ });
+
+ test('ValidateCheckRuns no failure for status completed and neutral conclusion.', () {
+ githubService.checkRunsData = neutralCheckRunsMock;
+ final Future<List<github.CheckRun>> checkRunFuture = githubService.getCheckRuns(slug, 'ref');
+ bool allSuccess = true;
+
+ checkRunFuture.then((checkRuns) {
+ expect(ciSuccessful.validateCheckRuns(slug, checkRuns, failures, allSuccess), isTrue);
+ expect(failures, isEmpty);
+ });
+ });
+
+ test('ValidateCheckRuns failure detected on status completed no neutral conclusion.', () {
+ githubService.checkRunsData = failedCheckRunsMock;
+ final Future<List<github.CheckRun>> checkRunFuture = githubService.getCheckRuns(slug, 'ref');
+ bool allSuccess = true;
+
+ checkRunFuture.then((checkRuns) {
+ expect(ciSuccessful.validateCheckRuns(slug, checkRuns, failures, allSuccess), isFalse);
+ expect(failures, isNotEmpty);
+ expect(failures.length, 1);
+ });
+ });
+
+ test('ValidateCheckRuns succes with multiple successful check runs.', () {
+ githubService.checkRunsData = multipleCheckRunsMock;
+ final Future<List<github.CheckRun>> checkRunFuture = githubService.getCheckRuns(slug, 'ref');
+ bool allSuccess = true;
+
+ checkRunFuture.then((checkRuns) {
+ expect(ciSuccessful.validateCheckRuns(slug, checkRuns, failures, allSuccess), isTrue);
+ expect(failures, isEmpty);
+ });
+ });
+
+ test('ValidateCheckRuns failed with multiple check runs.', () {
+ githubService.checkRunsData = multipleCheckRunsWithFailureMock;
+ final Future<List<github.CheckRun>> checkRunFuture = githubService.getCheckRuns(slug, 'ref');
+ bool allSuccess = true;
+
+ checkRunFuture.then((checkRuns) {
+ expect(ciSuccessful.validateCheckRuns(slug, checkRuns, failures, allSuccess), isFalse);
+ expect(failures, isNotEmpty);
+ expect(failures.length, 1);
+ });
+ });
+
+ test('ValidateCheckRuns allSucces false but no failures recorded.', () {
+ /// This test just checks that a checkRun that has not yet completed and
+ /// does not cause failure is a candidate to be temporarily ignored.
+ githubService.checkRunsData = inprogressAndNotFailedCheckRunMock;
+ final Future<List<github.CheckRun>> checkRunFuture = githubService.getCheckRuns(slug, 'ref');
+ bool allSuccess = true;
+
+ checkRunFuture.then((checkRuns) {
+ expect(ciSuccessful.validateCheckRuns(slug, checkRuns, failures, allSuccess), isFalse);
+ expect(failures, isEmpty);
+ });
+ });
+
+ test('ValidateCheckRuns allSuccess false is preserved.', () {
+ githubService.checkRunsData = multipleCheckRunsWithFailureMock;
+ final Future<List<github.CheckRun>> checkRunFuture = githubService.getCheckRuns(slug, 'ref');
+ bool allSuccess = false;
+
+ checkRunFuture.then((checkRuns) {
+ expect(ciSuccessful.validateCheckRuns(slug, checkRuns, failures, allSuccess), isFalse);
+ expect(failures, isNotEmpty);
+ expect(failures.length, 1);
+ });
+ });
+ });
+
+ group('validateStatuses', () {
+ test('Validate successful statuses show as successful.', () {
+ final List<ContextNode> contextNodeList = _getContextNodeListFromJson(repositoryStatusesMock);
+ bool allSuccess = true;
+
+ /// The status must be uppercase as the original code is expecting this.
+ _convertContextNodeStatuses(contextNodeList);
+ expect(ciSuccessful.validateStatuses(slug, [], contextNodeList, failures, allSuccess), isTrue);
+ expect(failures, isEmpty);
+ });
+
+ test('Validate statuses that are not successful but do not cause failure.', () {
+ final List<ContextNode> contextNodeList = _getContextNodeListFromJson(failedAuthorsStatusesMock);
+ bool allSuccess = true;
+
+ final List<String> labelNames = [];
+ labelNames.add('warning: land on red to fix tree breakage');
+ labelNames.add('Other label');
+
+ _convertContextNodeStatuses(contextNodeList);
+ expect(ciSuccessful.validateStatuses(slug, labelNames, contextNodeList, failures, allSuccess), isTrue);
+ expect(failures, isEmpty);
+ });
+
+ test('Validate failure statuses do not cause failure with not in authors control.', () {
+ final List<ContextNode> contextNodeList = _getContextNodeListFromJson(failedAuthorsStatusesMock);
+ bool allSuccess = true;
+
+ final List<String> labelNames = [];
+ labelNames.add('Compelling label');
+ labelNames.add('Another Compelling label');
+
+ _convertContextNodeStatuses(contextNodeList);
+ expect(ciSuccessful.validateStatuses(slug, labelNames, contextNodeList, failures, allSuccess), isFalse);
+ expect(failures, isEmpty);
+ });
+
+ test('Validate failure statuses cause failures with not in authors control.', () {
+ final List<ContextNode> contextNodeList = _getContextNodeListFromJson(failedNonAuthorsStatusesMock);
+ bool allSuccess = true;
+
+ final List<String> labelNames = [];
+ labelNames.add('Compelling label');
+ labelNames.add('Another Compelling label');
+
+ _convertContextNodeStatuses(contextNodeList);
+ expect(ciSuccessful.validateStatuses(slug, labelNames, contextNodeList, failures, allSuccess), isFalse);
+ expect(failures, isNotEmpty);
+ expect(failures.length, 2);
+ });
+
+ test('Validate failure statuses cause failures and preserves false allSuccess.', () {
+ final List<ContextNode> contextNodeList = _getContextNodeListFromJson(failedNonAuthorsStatusesMock);
+ bool allSuccess = false;
+
+ final List<String> labelNames = [];
+ labelNames.add('Compelling label');
+ labelNames.add('Another Compelling label');
+
+ _convertContextNodeStatuses(contextNodeList);
+ expect(ciSuccessful.validateStatuses(slug, labelNames, contextNodeList, failures, allSuccess), isFalse);
+ expect(failures, isNotEmpty);
+ expect(failures.length, 2);
+ });
+ });
+
+ group('validateTreeStatusIsSet', () {
+ test('Validate tree status is set contains slug.', () {
+ slug = github.RepositorySlug('octocat', 'flutter');
+ final List<ContextNode> contextNodeList = _getContextNodeListFromJson(repositoryStatusesMock);
+
+ /// The status must be uppercase as the original code is expecting this.
+ _convertContextNodeStatuses(contextNodeList);
+ ciSuccessful.validateTreeStatusIsSet(slug, contextNodeList, failures);
+ expect(failures, isEmpty);
+ });
+
+ test('Validate tree status is set does not contain slug.', () {
+ slug = github.RepositorySlug('octocat', 'infra');
+ final List<ContextNode> contextNodeList = _getContextNodeListFromJson(repositoryStatusesMock);
+
+ /// The status must be uppercase as the original code is expecting this.
+ _convertContextNodeStatuses(contextNodeList);
+ ciSuccessful.validateTreeStatusIsSet(slug, contextNodeList, failures);
+ expect(failures, isEmpty);
+ });
+
+ test('Validate tree status is set but context does not match slug.', () {
+ slug = github.RepositorySlug('flutter', 'flutter');
+ final List<ContextNode> contextNodeList = _getContextNodeListFromJson(repositoryStatusesNonLuciFlutterMock);
+
+ /// The status must be uppercase as the original code is expecting this.
+ _convertContextNodeStatuses(contextNodeList);
+ ciSuccessful.validateTreeStatusIsSet(slug, contextNodeList, failures);
+ expect(failures, isNotEmpty);
+ expect(failures.length, 1);
+ });
+ });
+
+ group('validate', () {
+ test('Commit has a null status, no statuses to verify.', () {
+ final Map<String, dynamic> queryResultJsonDecode =
+ jsonDecode(nullStatusCommitRepositoryJson) as Map<String, dynamic>;
+ final QueryResult queryResult = QueryResult.fromJson(queryResultJsonDecode);
+ expect(queryResult, isNotNull);
+ final PullRequest pr = queryResult.repository!.pullRequest!;
+ expect(pr, isNotNull);
+ final Commit commit = pr.commits!.nodes!.single.commit!;
+ expect(commit, isNotNull);
+ expect(commit.status, isNull);
+
+ final github.PullRequest npr = generatePullRequest(labelName: 'needs tests');
+ githubService.checkRunsData = checkRunsMock;
+
+ ciSuccessful.validate(queryResult, npr).then((value) {
+ // No failure.
+ expect(true, value.result);
+ // Remove label.
+ expect((value.action == Action.REMOVE_LABEL), isTrue);
+ expect(value.message,
+ '- The status or check suite [tree status luci-flutter](https://flutter-dashboard.appspot.com/#/build) has failed. Please fix the issues identified (or deflake) before re-applying this label.\n');
+ });
+ });
+
+ test('Commit has statuses to verify, action remove label, no message.', () {
+ final Map<String, dynamic> queryResultJsonDecode =
+ jsonDecode(nonNullStatusSUCCESSCommitRepositoryJson) as Map<String, dynamic>;
+ final QueryResult queryResult = QueryResult.fromJson(queryResultJsonDecode);
+ expect(queryResult, isNotNull);
+ final PullRequest pr = queryResult.repository!.pullRequest!;
+ expect(pr, isNotNull);
+ final Commit commit = pr.commits!.nodes!.single.commit!;
+ expect(commit, isNotNull);
+ expect(commit.status, isNotNull);
+
+ final github.PullRequest npr = generatePullRequest(labelName: 'needs tests');
+ githubService.checkRunsData = checkRunsMock;
+
+ ciSuccessful.validate(queryResult, npr).then((value) {
+ // No failure.
+ expect(value.result, isTrue);
+ // Remove label.
+ expect((value.action == Action.REMOVE_LABEL), isTrue);
+ expect(value.message, isEmpty);
+ });
+ });
+
+ test('Commit has statuses to verify, action ignore failure, no message.', () {
+ final Map<String, dynamic> queryResultJsonDecode =
+ jsonDecode(nonNullStatusFAILURECommitRepositoryJson) as Map<String, dynamic>;
+ final QueryResult queryResult = QueryResult.fromJson(queryResultJsonDecode);
+ expect(queryResult, isNotNull);
+ final PullRequest pr = queryResult.repository!.pullRequest!;
+ expect(pr, isNotNull);
+ final Commit commit = pr.commits!.nodes!.single.commit!;
+ expect(commit, isNotNull);
+ expect(commit.status, isNotNull);
+
+ final github.PullRequest npr = generatePullRequest(labelName: 'warning: land on red to fix tree breakage');
+ githubService.checkRunsData = checkRunsMock;
+
+ ciSuccessful.validate(queryResult, npr).then((value) {
+ // No failure.
+ expect(value.result, isTrue);
+ // Remove label.
+ expect((value.action == Action.IGNORE_FAILURE), isTrue);
+ expect(value.message, isEmpty);
+ });
+ });
+
+ test('Commit has statuses to verify, action failure, no message.', () {
+ final Map<String, dynamic> queryResultJsonDecode =
+ jsonDecode(nonNullStatusFAILURECommitRepositoryJson) as Map<String, dynamic>;
+ final QueryResult queryResult = QueryResult.fromJson(queryResultJsonDecode);
+ expect(queryResult, isNotNull);
+ final PullRequest pr = queryResult.repository!.pullRequest!;
+ expect(pr, isNotNull);
+ final Commit commit = pr.commits!.nodes!.single.commit!;
+ expect(commit, isNotNull);
+ expect(commit.status, isNotNull);
+
+ final github.PullRequest npr = generatePullRequest();
+ githubService.checkRunsData = checkRunsMock;
+
+ ciSuccessful.validate(queryResult, npr).then((value) {
+ // No failure.
+ expect(false, value.result);
+ // Remove label.
+ expect((value.action == Action.IGNORE_TEMPORARILY), isTrue);
+ expect(value.message, isEmpty);
+ });
+ });
+ });
+
+ group('Validate empty message is not returned.', () {
+ setUp(() {
+ githubService = FakeGithubService(client: MockGitHub());
+ config = FakeConfig(githubService: githubService);
+ ciSuccessful = CiSuccessful(config: config);
+ slug = github.RepositorySlug('flutter', 'cocoon');
+ });
+
+ test('returns correct message when validation fails', () async {
+ PullRequestHelper flutterRequest = PullRequestHelper(
+ prNumber: 0,
+ lastCommitHash: oid,
+ reviews: <PullRequestReviewHelper>[],
+ );
+
+ githubService.checkRunsData = failedCheckRunsMock;
+ final github.PullRequest pullRequest = generatePullRequest(prNumber: 0, repoName: slug.name);
+ QueryResult queryResult = createQueryResult(flutterRequest);
+
+ final ValidationResult validationResult = await ciSuccessful.validate(queryResult, pullRequest);
+
+ expect(validationResult.result, false);
+ expect(validationResult.message,
+ '- The status or check suite [failed_checkrun](https://example.com) has failed. Please fix the issues identified (or deflake) before re-applying this label.\n');
+ });
});
}
diff --git a/auto_submit/test/validations/ci_successful_test_data.dart b/auto_submit/test/validations/ci_successful_test_data.dart
new file mode 100644
index 0000000..9676629
--- /dev/null
+++ b/auto_submit/test/validations/ci_successful_test_data.dart
@@ -0,0 +1,138 @@
+// Copyright 2022 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/// Constants used for testing in ci_successful_test.dart.
+
+const String nullStatusCommitRepositoryJson = """
+ {
+ "repository": {
+ "pullRequest": {
+ "author": {
+ "login": "author1"
+ },
+ "authorAssociation": "MEMBER",
+ "id": "PR_kwDOA8VHis43rs4_",
+ "title": "[dependabot] Remove human reviewers",
+ "commits": {
+ "nodes":[
+ {
+ "commit": {
+ "abbreviatedOid": "4009ecc",
+ "oid": "4009ecc0b6dbf5cb19cb97472147063e7368ec10",
+ "committedDate": "2022-05-11T22:35:02Z",
+ "pushedDate": "2022-05-11T22:35:03Z",
+ "status": null
+ }
+ }
+ ]
+ },
+ "reviews": {
+ "nodes": [
+ {
+ "author": {
+ "login": "keyonghan"
+ },
+ "authorAssociation": "MEMBER",
+ "state": "APPROVED"
+ }
+ ]
+ }
+ }
+ }
+ }
+ """;
+
+const String nonNullStatusSUCCESSCommitRepositoryJson = """
+ {
+ "repository": {
+ "pullRequest": {
+ "author": {
+ "login": "author1"
+ },
+ "authorAssociation": "MEMBER",
+ "id": "PR_kwDOA8VHis43rs4_",
+ "title": "[dependabot] Remove human reviewers",
+ "commits": {
+ "nodes":[
+ {
+ "commit": {
+ "abbreviatedOid": "4009ecc",
+ "oid": "4009ecc0b6dbf5cb19cb97472147063e7368ec10",
+ "committedDate": "2022-05-11T22:35:02Z",
+ "pushedDate": "2022-05-11T22:35:03Z",
+ "status": {
+ "contexts":[
+ {
+ "context":"luci-flutter",
+ "state":"SUCCESS",
+ "targetUrl":"https://ci.example.com/1000/output"
+ }
+ ]
+ }
+ }
+ }
+ ]
+ },
+ "reviews": {
+ "nodes": [
+ {
+ "author": {
+ "login": "keyonghan"
+ },
+ "authorAssociation": "MEMBER",
+ "state": "APPROVED"
+ }
+ ]
+ }
+ }
+ }
+ }
+ """;
+
+const String nonNullStatusFAILURECommitRepositoryJson = """
+ {
+ "repository": {
+ "pullRequest": {
+ "author": {
+ "login": "author1"
+ },
+ "authorAssociation": "MEMBER",
+ "id": "PR_kwDOA8VHis43rs4_",
+ "title": "[dependabot] Remove human reviewers",
+ "commits": {
+ "nodes":[
+ {
+ "commit": {
+ "abbreviatedOid": "4009ecc",
+ "oid": "4009ecc0b6dbf5cb19cb97472147063e7368ec10",
+ "committedDate": "2022-05-11T22:35:02Z",
+ "pushedDate": "2022-05-11T22:35:03Z",
+ "status": {
+ "contexts":[
+ {
+ "context":"luci-flutter",
+ "state":"FAILURE",
+ "targetUrl":"https://ci.example.com/1000/output"
+ }
+ ]
+ }
+ }
+ }
+ ]
+ },
+ "reviews": {
+ "nodes": [
+ {
+ "author": {
+ "login": "keyonghan"
+ },
+ "authorAssociation": "MEMBER",
+ "state": "APPROVED"
+ }
+ ]
+ }
+ }
+ }
+ }
+ """;
diff --git a/dashboard/pubspec.lock b/dashboard/pubspec.lock
index 962a212..ca8ca36 100644
--- a/dashboard/pubspec.lock
+++ b/dashboard/pubspec.lock
@@ -126,7 +126,7 @@
name: clock
url: "https://pub.dartlang.org"
source: hosted
- version: "1.1.1"
+ version: "1.1.0"
code_builder:
dependency: transitive
description:
@@ -168,7 +168,7 @@
name: fake_async
url: "https://pub.dartlang.org"
source: hosted
- version: "1.3.1"
+ version: "1.3.0"
file:
dependency: transitive
description:
@@ -489,7 +489,7 @@
name: term_glyph
url: "https://pub.dartlang.org"
source: hosted
- version: "1.2.1"
+ version: "1.2.0"
test_api:
dependency: transitive
description:
